2024-11-07 15:22:25,790 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-07 15:22:25,804 main DEBUG Took 0.012055 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-07 15:22:25,805 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-07 15:22:25,805 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-07 15:22:25,807 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-07 15:22:25,808 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:22:25,817 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-07 15:22:25,833 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:22:25,834 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:22:25,835 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:22:25,836 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:22:25,837 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:22:25,837 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:22:25,838 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:22:25,839 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:22:25,839 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:22:25,840 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:22:25,841 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:22:25,842 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:22:25,843 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:22:25,843 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:22:25,844 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:22:25,844 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:22:25,845 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:22:25,845 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:22:25,847 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:22:25,847 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:22:25,848 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:22:25,848 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:22:25,849 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:22:25,849 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:22:25,850 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:22:25,851 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-07 15:22:25,852 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:22:25,854 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-07 15:22:25,856 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-07 15:22:25,857 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-07 15:22:25,858 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-07 15:22:25,859 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-07 15:22:25,870 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-07 15:22:25,874 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-07 15:22:25,877 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-07 15:22:25,878 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-07 15:22:25,879 main DEBUG createAppenders(={Console}) 2024-11-07 15:22:25,882 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-07 15:22:25,884 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-07 15:22:25,885 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-07 15:22:25,886 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-07 15:22:25,887 main DEBUG OutputStream closed 2024-11-07 15:22:25,888 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-07 15:22:25,889 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-07 15:22:25,894 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-07 15:22:25,999 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-07 15:22:26,002 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-07 15:22:26,003 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-07 15:22:26,005 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-07 15:22:26,006 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-07 15:22:26,007 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-07 15:22:26,007 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-07 15:22:26,008 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-07 15:22:26,008 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-07 15:22:26,009 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-07 15:22:26,009 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-07 15:22:26,009 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-07 15:22:26,010 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-07 15:22:26,010 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-07 15:22:26,011 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-07 15:22:26,011 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-07 15:22:26,012 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-07 15:22:26,013 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-07 15:22:26,016 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07 15:22:26,016 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-07 15:22:26,017 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-07 15:22:26,018 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-07T15:22:26,384 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2 2024-11-07 15:22:26,388 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-07 15:22:26,388 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07T15:22:26,400 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-07T15:22:26,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-07T15:22:26,449 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/cluster_df3b216e-6ff7-ba4d-e7a0-e36a33a739e6, deleteOnExit=true 2024-11-07T15:22:26,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-07T15:22:26,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/test.cache.data in system properties and HBase conf 2024-11-07T15:22:26,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/hadoop.tmp.dir in system properties and HBase conf 2024-11-07T15:22:26,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/hadoop.log.dir in system properties and HBase conf 2024-11-07T15:22:26,453 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-07T15:22:26,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-07T15:22:26,455 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-07T15:22:26,538 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-07T15:22:26,616 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T15:22:26,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-07T15:22:26,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-07T15:22:26,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-07T15:22:26,622 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T15:22:26,622 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-07T15:22:26,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-07T15:22:26,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T15:22:26,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T15:22:26,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-07T15:22:26,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/nfs.dump.dir in system properties and HBase conf 2024-11-07T15:22:26,625 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/java.io.tmpdir in system properties and HBase conf 2024-11-07T15:22:26,625 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T15:22:26,625 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-07T15:22:26,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-07T15:22:27,856 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-07T15:22:27,923 INFO [Time-limited test {}] log.Log(170): Logging initialized @3220ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-07T15:22:27,990 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T15:22:28,052 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T15:22:28,072 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T15:22:28,072 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T15:22:28,073 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-07T15:22:28,083 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T15:22:28,086 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/hadoop.log.dir/,AVAILABLE} 2024-11-07T15:22:28,087 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T15:22:28,278 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/java.io.tmpdir/jetty-localhost-39879-hadoop-hdfs-3_4_1-tests_jar-_-any-9190683686747600693/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T15:22:28,290 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:39879} 2024-11-07T15:22:28,290 INFO [Time-limited test {}] server.Server(415): Started @3587ms 2024-11-07T15:22:28,837 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T15:22:28,844 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T15:22:28,846 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T15:22:28,846 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T15:22:28,846 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-07T15:22:28,847 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/hadoop.log.dir/,AVAILABLE} 2024-11-07T15:22:28,847 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T15:22:28,948 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dfb1097{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/java.io.tmpdir/jetty-localhost-44223-hadoop-hdfs-3_4_1-tests_jar-_-any-7610990474350491165/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T15:22:28,948 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:44223} 2024-11-07T15:22:28,949 INFO [Time-limited test {}] server.Server(415): Started @4246ms 2024-11-07T15:22:28,998 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T15:22:30,150 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/cluster_df3b216e-6ff7-ba4d-e7a0-e36a33a739e6/data/data2/current/BP-750064135-172.17.0.2-1730992947237/current, will proceed with Du for space computation calculation, 2024-11-07T15:22:30,150 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/cluster_df3b216e-6ff7-ba4d-e7a0-e36a33a739e6/data/data1/current/BP-750064135-172.17.0.2-1730992947237/current, will proceed with Du for space computation calculation, 2024-11-07T15:22:30,179 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T15:22:30,222 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd4767049a3e3fe42 with lease ID 0xcc3acc9383217c91: Processing first storage report for DS-967daab8-f425-4b96-ad48-fdabdac740e1 from datanode DatanodeRegistration(127.0.0.1:44095, datanodeUuid=2bee1f03-7b8c-47c3-83e2-8866e9b3fe97, infoPort=43273, infoSecurePort=0, ipcPort=35471, storageInfo=lv=-57;cid=testClusterID;nsid=1502240258;c=1730992947237) 2024-11-07T15:22:30,224 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd4767049a3e3fe42 with lease ID 0xcc3acc9383217c91: from storage DS-967daab8-f425-4b96-ad48-fdabdac740e1 node DatanodeRegistration(127.0.0.1:44095, datanodeUuid=2bee1f03-7b8c-47c3-83e2-8866e9b3fe97, infoPort=43273, infoSecurePort=0, ipcPort=35471, storageInfo=lv=-57;cid=testClusterID;nsid=1502240258;c=1730992947237), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-07T15:22:30,224 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd4767049a3e3fe42 with lease ID 0xcc3acc9383217c91: Processing first storage report for DS-5a06367e-4fbb-417f-8ece-202077ea153b from datanode DatanodeRegistration(127.0.0.1:44095, datanodeUuid=2bee1f03-7b8c-47c3-83e2-8866e9b3fe97, infoPort=43273, infoSecurePort=0, ipcPort=35471, storageInfo=lv=-57;cid=testClusterID;nsid=1502240258;c=1730992947237) 2024-11-07T15:22:30,224 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd4767049a3e3fe42 with lease ID 0xcc3acc9383217c91: from storage DS-5a06367e-4fbb-417f-8ece-202077ea153b node DatanodeRegistration(127.0.0.1:44095, datanodeUuid=2bee1f03-7b8c-47c3-83e2-8866e9b3fe97, infoPort=43273, infoSecurePort=0, ipcPort=35471, storageInfo=lv=-57;cid=testClusterID;nsid=1502240258;c=1730992947237), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T15:22:30,296 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2 2024-11-07T15:22:30,359 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/cluster_df3b216e-6ff7-ba4d-e7a0-e36a33a739e6/zookeeper_0, clientPort=54223, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/cluster_df3b216e-6ff7-ba4d-e7a0-e36a33a739e6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/cluster_df3b216e-6ff7-ba4d-e7a0-e36a33a739e6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T15:22:30,368 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54223 2024-11-07T15:22:30,377 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T15:22:30,379 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T15:22:30,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741825_1001 (size=7) 2024-11-07T15:22:30,998 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938 with version=8 2024-11-07T15:22:30,998 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/hbase-staging 2024-11-07T15:22:31,084 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-07T15:22:31,311 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7b3e4e8e12ef:0 server-side Connection retries=45 2024-11-07T15:22:31,319 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T15:22:31,319 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T15:22:31,324 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T15:22:31,324 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T15:22:31,324 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T15:22:31,447 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-07T15:22:31,499 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-07T15:22:31,507 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-07T15:22:31,510 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T15:22:31,536 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 31038 (auto-detected) 2024-11-07T15:22:31,538 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-07T15:22:31,560 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46811 2024-11-07T15:22:31,580 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46811 connecting to ZooKeeper ensemble=127.0.0.1:54223 2024-11-07T15:22:31,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:468110x0, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T15:22:31,679 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46811-0x10115e447f30000 connected 2024-11-07T15:22:31,780 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T15:22:31,782 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T15:22:31,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T15:22:31,794 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938, hbase.cluster.distributed=false 2024-11-07T15:22:31,819 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T15:22:31,823 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46811 2024-11-07T15:22:31,824 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46811 2024-11-07T15:22:31,826 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46811 2024-11-07T15:22:31,836 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46811 2024-11-07T15:22:31,837 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46811 2024-11-07T15:22:31,940 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7b3e4e8e12ef:0 server-side Connection retries=45 2024-11-07T15:22:31,942 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T15:22:31,943 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T15:22:31,943 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T15:22:31,944 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T15:22:31,944 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T15:22:31,947 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T15:22:31,950 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T15:22:31,951 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33849 2024-11-07T15:22:31,953 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33849 connecting to ZooKeeper ensemble=127.0.0.1:54223 2024-11-07T15:22:31,955 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T15:22:31,959 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T15:22:31,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:338490x0, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T15:22:31,976 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:338490x0, quorum=127.0.0.1:54223, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T15:22:31,976 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33849-0x10115e447f30001 connected 2024-11-07T15:22:31,981 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T15:22:31,990 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T15:22:31,994 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T15:22:32,000 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T15:22:32,001 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33849 2024-11-07T15:22:32,001 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33849 2024-11-07T15:22:32,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33849 2024-11-07T15:22:32,007 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33849 2024-11-07T15:22:32,007 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33849 2024-11-07T15:22:32,021 DEBUG [M:0;7b3e4e8e12ef:46811 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7b3e4e8e12ef:46811 2024-11-07T15:22:32,022 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7b3e4e8e12ef,46811,1730992951165 2024-11-07T15:22:32,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T15:22:32,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T15:22:32,041 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7b3e4e8e12ef,46811,1730992951165 2024-11-07T15:22:32,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T15:22:32,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:22:32,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:22:32,103 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T15:22:32,105 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7b3e4e8e12ef,46811,1730992951165 from backup master directory 2024-11-07T15:22:32,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7b3e4e8e12ef,46811,1730992951165 2024-11-07T15:22:32,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T15:22:32,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T15:22:32,189 WARN [master/7b3e4e8e12ef:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T15:22:32,190 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7b3e4e8e12ef,46811,1730992951165 2024-11-07T15:22:32,195 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-07T15:22:32,197 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-07T15:22:32,248 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/hbase.id] with ID: 602c0594-20fc-4ac0-9c8a-0e1a48d3c68a 2024-11-07T15:22:32,248 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/.tmp/hbase.id 2024-11-07T15:22:32,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741826_1002 (size=42) 2024-11-07T15:22:32,258 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/.tmp/hbase.id]:[hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/hbase.id] 2024-11-07T15:22:32,304 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T15:22:32,308 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-07T15:22:32,326 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-07T15:22:32,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:22:32,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:22:32,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741827_1003 (size=196) 2024-11-07T15:22:32,385 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:22:32,388 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T15:22:32,401 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:32,405 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T15:22:32,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741828_1004 (size=1189) 2024-11-07T15:22:32,854 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store 2024-11-07T15:22:32,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741829_1005 (size=34) 2024-11-07T15:22:33,279 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-07T15:22:33,282 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:22:33,283 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T15:22:33,283 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T15:22:33,283 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T15:22:33,285 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T15:22:33,285 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T15:22:33,285 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T15:22:33,286 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730992953283Disabling compacts and flushes for region at 1730992953283Disabling writes for close at 1730992953285 (+2 ms)Writing region close event to WAL at 1730992953285Closed at 1730992953285 2024-11-07T15:22:33,288 WARN [master/7b3e4e8e12ef:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/.initializing 2024-11-07T15:22:33,289 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/WALs/7b3e4e8e12ef,46811,1730992951165 2024-11-07T15:22:33,298 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T15:22:33,318 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7b3e4e8e12ef%2C46811%2C1730992951165, suffix=, logDir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/WALs/7b3e4e8e12ef,46811,1730992951165, archiveDir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/oldWALs, maxLogs=10 2024-11-07T15:22:33,349 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/WALs/7b3e4e8e12ef,46811,1730992951165/7b3e4e8e12ef%2C46811%2C1730992951165.1730992953324, exclude list is [], retry=0 2024-11-07T15:22:33,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44095,DS-967daab8-f425-4b96-ad48-fdabdac740e1,DISK] 2024-11-07T15:22:33,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-07T15:22:33,413 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/WALs/7b3e4e8e12ef,46811,1730992951165/7b3e4e8e12ef%2C46811%2C1730992951165.1730992953324 2024-11-07T15:22:33,414 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43273:43273)] 2024-11-07T15:22:33,415 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:22:33,416 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:22:33,421 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:22:33,423 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:22:33,473 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:22:33,505 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T15:22:33,510 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:33,514 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:22:33,514 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:22:33,519 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T15:22:33,519 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:33,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:22:33,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:22:33,525 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T15:22:33,525 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:33,527 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:22:33,527 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:22:33,531 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T15:22:33,531 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:33,533 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:22:33,533 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:22:33,538 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:22:33,540 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:22:33,546 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:22:33,547 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:22:33,552 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T15:22:33,557 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:22:33,562 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T15:22:33,563 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67102524, jitterRate=-9.447336196899414E-5}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T15:22:33,570 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1730992953439Initializing all the Stores at 1730992953442 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730992953442Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730992953443 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730992953443Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730992953444 (+1 ms)Cleaning up temporary data from old regions at 1730992953547 (+103 ms)Region opened successfully at 1730992953570 (+23 ms) 2024-11-07T15:22:33,572 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T15:22:33,616 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f95140, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7b3e4e8e12ef/172.17.0.2:0 2024-11-07T15:22:33,655 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-07T15:22:33,668 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-07T15:22:33,668 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-07T15:22:33,671 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T15:22:33,672 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-07T15:22:33,677 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T15:22:33,677 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T15:22:33,709 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T15:22:33,720 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T15:22:33,830 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-07T15:22:33,833 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T15:22:33,835 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T15:22:33,841 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-07T15:22:33,843 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T15:22:33,847 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T15:22:33,859 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-07T15:22:33,861 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T15:22:33,876 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T15:22:33,897 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T15:22:33,904 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T15:22:33,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T15:22:33,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T15:22:33,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:22:33,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:22:33,918 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7b3e4e8e12ef,46811,1730992951165, sessionid=0x10115e447f30000, setting cluster-up flag (Was=false) 2024-11-07T15:22:33,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:22:33,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:22:33,978 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T15:22:33,980 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7b3e4e8e12ef,46811,1730992951165 2024-11-07T15:22:34,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:22:34,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:22:34,041 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T15:22:34,043 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7b3e4e8e12ef,46811,1730992951165 2024-11-07T15:22:34,050 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-07T15:22:34,114 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(746): ClusterId : 602c0594-20fc-4ac0-9c8a-0e1a48d3c68a 2024-11-07T15:22:34,118 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T15:22:34,128 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T15:22:34,128 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T15:22:34,139 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T15:22:34,144 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T15:22:34,145 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57d3e60c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7b3e4e8e12ef/172.17.0.2:0 2024-11-07T15:22:34,152 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-07T15:22:34,159 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7b3e4e8e12ef:33849 2024-11-07T15:22:34,161 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T15:22:34,164 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T15:22:34,164 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T15:22:34,164 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T15:22:34,167 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(2659): reportForDuty to master=7b3e4e8e12ef,46811,1730992951165 with port=33849, startcode=1730992951906 2024-11-07T15:22:34,169 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7b3e4e8e12ef,46811,1730992951165 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T15:22:34,180 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7b3e4e8e12ef:0, corePoolSize=5, maxPoolSize=5 2024-11-07T15:22:34,180 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7b3e4e8e12ef:0, corePoolSize=5, maxPoolSize=5 2024-11-07T15:22:34,181 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7b3e4e8e12ef:0, corePoolSize=5, maxPoolSize=5 2024-11-07T15:22:34,181 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7b3e4e8e12ef:0, corePoolSize=5, maxPoolSize=5 2024-11-07T15:22:34,181 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7b3e4e8e12ef:0, corePoolSize=10, maxPoolSize=10 2024-11-07T15:22:34,182 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7b3e4e8e12ef:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:22:34,182 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7b3e4e8e12ef:0, corePoolSize=2, maxPoolSize=2 2024-11-07T15:22:34,182 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7b3e4e8e12ef:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:22:34,183 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T15:22:34,194 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730992984194 2024-11-07T15:22:34,196 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T15:22:34,196 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T15:22:34,196 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-07T15:22:34,197 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T15:22:34,202 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T15:22:34,202 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T15:22:34,203 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T15:22:34,203 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T15:22:34,210 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:34,211 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T15:22:34,217 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:34,222 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T15:22:34,224 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T15:22:34,224 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T15:22:34,227 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T15:22:34,228 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T15:22:34,232 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7b3e4e8e12ef:0:becomeActiveMaster-HFileCleaner.large.0-1730992954229,5,FailOnTimeoutGroup] 2024-11-07T15:22:34,234 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7b3e4e8e12ef:0:becomeActiveMaster-HFileCleaner.small.0-1730992954233,5,FailOnTimeoutGroup] 2024-11-07T15:22:34,234 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:34,234 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T15:22:34,235 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57945, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T15:22:34,235 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:34,236 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:34,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741831_1007 (size=1321) 2024-11-07T15:22:34,243 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46811 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:34,246 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46811 {}] master.ServerManager(517): Registering regionserver=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:34,262 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938 2024-11-07T15:22:34,262 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41161 2024-11-07T15:22:34,262 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T15:22:34,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T15:22:34,273 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] zookeeper.ZKUtil(111): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:34,274 WARN [RS:0;7b3e4e8e12ef:33849 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T15:22:34,274 INFO [RS:0;7b3e4e8e12ef:33849 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T15:22:34,274 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/WALs/7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:34,276 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7b3e4e8e12ef,33849,1730992951906] 2024-11-07T15:22:34,299 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T15:22:34,320 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T15:22:34,329 INFO [RS:0;7b3e4e8e12ef:33849 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T15:22:34,330 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:34,331 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T15:22:34,337 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T15:22:34,338 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:34,339 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:22:34,339 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7b3e4e8e12ef:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:22:34,339 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7b3e4e8e12ef:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:22:34,339 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:22:34,339 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7b3e4e8e12ef:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:22:34,340 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7b3e4e8e12ef:0, corePoolSize=2, maxPoolSize=2 2024-11-07T15:22:34,340 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7b3e4e8e12ef:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:22:34,340 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7b3e4e8e12ef:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:22:34,340 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7b3e4e8e12ef:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:22:34,340 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7b3e4e8e12ef:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:22:34,340 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7b3e4e8e12ef:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:22:34,341 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7b3e4e8e12ef:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:22:34,341 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7b3e4e8e12ef:0, corePoolSize=3, maxPoolSize=3 2024-11-07T15:22:34,341 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0, corePoolSize=3, maxPoolSize=3 2024-11-07T15:22:34,343 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:34,343 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:34,343 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:34,344 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:34,344 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:34,344 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b3e4e8e12ef,33849,1730992951906-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T15:22:34,365 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T15:22:34,367 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b3e4e8e12ef,33849,1730992951906-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:34,367 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:34,367 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.Replication(171): 7b3e4e8e12ef,33849,1730992951906 started 2024-11-07T15:22:34,392 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:34,393 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1482): Serving as 7b3e4e8e12ef,33849,1730992951906, RpcServer on 7b3e4e8e12ef/172.17.0.2:33849, sessionid=0x10115e447f30001 2024-11-07T15:22:34,394 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T15:22:34,394 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:34,394 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b3e4e8e12ef,33849,1730992951906' 2024-11-07T15:22:34,394 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T15:22:34,396 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T15:22:34,397 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T15:22:34,397 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T15:22:34,398 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:34,398 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b3e4e8e12ef,33849,1730992951906' 2024-11-07T15:22:34,398 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T15:22:34,399 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T15:22:34,400 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T15:22:34,400 INFO [RS:0;7b3e4e8e12ef:33849 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T15:22:34,400 INFO [RS:0;7b3e4e8e12ef:33849 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T15:22:34,504 INFO [RS:0;7b3e4e8e12ef:33849 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T15:22:34,507 INFO [RS:0;7b3e4e8e12ef:33849 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7b3e4e8e12ef%2C33849%2C1730992951906, suffix=, logDir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/WALs/7b3e4e8e12ef,33849,1730992951906, archiveDir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/oldWALs, maxLogs=32 2024-11-07T15:22:34,523 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/WALs/7b3e4e8e12ef,33849,1730992951906/7b3e4e8e12ef%2C33849%2C1730992951906.1730992954510, exclude list is [], retry=0 2024-11-07T15:22:34,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44095,DS-967daab8-f425-4b96-ad48-fdabdac740e1,DISK] 2024-11-07T15:22:34,531 INFO [RS:0;7b3e4e8e12ef:33849 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/WALs/7b3e4e8e12ef,33849,1730992951906/7b3e4e8e12ef%2C33849%2C1730992951906.1730992954510 2024-11-07T15:22:34,531 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43273:43273)] 2024-11-07T15:22:34,645 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-07T15:22:34,646 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938 2024-11-07T15:22:34,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741833_1009 (size=32) 2024-11-07T15:22:35,058 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:22:35,061 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T15:22:35,065 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T15:22:35,065 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:35,066 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:22:35,066 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T15:22:35,070 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T15:22:35,070 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:35,071 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:22:35,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T15:22:35,075 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T15:22:35,075 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:35,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:22:35,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T15:22:35,080 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T15:22:35,080 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:35,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:22:35,082 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T15:22:35,084 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740 2024-11-07T15:22:35,085 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740 2024-11-07T15:22:35,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T15:22:35,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T15:22:35,090 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:22:35,094 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T15:22:35,098 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T15:22:35,099 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61002690, jitterRate=-0.09098908305168152}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:22:35,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1730992955058Initializing all the Stores at 1730992955060 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730992955061 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730992955061Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730992955061Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730992955061Cleaning up temporary data from old regions at 1730992955089 (+28 ms)Region opened successfully at 1730992955101 (+12 ms) 2024-11-07T15:22:35,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T15:22:35,102 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T15:22:35,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T15:22:35,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T15:22:35,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T15:22:35,104 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T15:22:35,104 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730992955102Disabling compacts and flushes for region at 1730992955102Disabling writes for close at 1730992955102Writing region close event to WAL at 1730992955103 (+1 ms)Closed at 1730992955104 (+1 ms) 2024-11-07T15:22:35,108 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T15:22:35,108 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-07T15:22:35,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T15:22:35,125 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T15:22:35,129 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T15:22:35,282 DEBUG [7b3e4e8e12ef:46811 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-07T15:22:35,297 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:35,303 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b3e4e8e12ef,33849,1730992951906, state=OPENING 2024-11-07T15:22:35,367 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T15:22:35,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:22:35,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:22:35,420 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T15:22:35,420 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T15:22:35,424 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T15:22:35,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:22:35,607 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T15:22:35,610 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33485, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T15:22:35,623 INFO [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T15:22:35,623 INFO [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T15:22:35,624 INFO [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T15:22:35,627 INFO [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7b3e4e8e12ef%2C33849%2C1730992951906.meta, suffix=.meta, logDir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/WALs/7b3e4e8e12ef,33849,1730992951906, archiveDir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/oldWALs, maxLogs=32 2024-11-07T15:22:35,642 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/WALs/7b3e4e8e12ef,33849,1730992951906/7b3e4e8e12ef%2C33849%2C1730992951906.meta.1730992955629.meta, exclude list is [], retry=0 2024-11-07T15:22:35,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44095,DS-967daab8-f425-4b96-ad48-fdabdac740e1,DISK] 2024-11-07T15:22:35,649 INFO [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/WALs/7b3e4e8e12ef,33849,1730992951906/7b3e4e8e12ef%2C33849%2C1730992951906.meta.1730992955629.meta 2024-11-07T15:22:35,649 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43273:43273)] 2024-11-07T15:22:35,650 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:22:35,651 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T15:22:35,654 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T15:22:35,659 INFO [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T15:22:35,664 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T15:22:35,665 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:22:35,665 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T15:22:35,666 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T15:22:35,669 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T15:22:35,671 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T15:22:35,671 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:35,672 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:22:35,672 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T15:22:35,674 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T15:22:35,674 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:35,675 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:22:35,675 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T15:22:35,676 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T15:22:35,676 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:35,677 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:22:35,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T15:22:35,679 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T15:22:35,679 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:35,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:22:35,680 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T15:22:35,682 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740 2024-11-07T15:22:35,686 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740 2024-11-07T15:22:35,689 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T15:22:35,689 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T15:22:35,690 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:22:35,693 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T15:22:35,696 INFO [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69142269, jitterRate=0.030300095677375793}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:22:35,696 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T15:22:35,698 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1730992955666Writing region info on filesystem at 1730992955666Initializing all the Stores at 1730992955668 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730992955668Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730992955669 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730992955669Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730992955669Cleaning up temporary data from old regions at 1730992955689 (+20 ms)Running coprocessor post-open hooks at 1730992955696 (+7 ms)Region opened successfully at 1730992955698 (+2 ms) 2024-11-07T15:22:35,706 INFO [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1730992955596 2024-11-07T15:22:35,719 DEBUG [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T15:22:35,720 INFO [RS_OPEN_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T15:22:35,722 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:35,725 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b3e4e8e12ef,33849,1730992951906, state=OPEN 2024-11-07T15:22:35,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T15:22:35,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T15:22:35,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T15:22:35,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T15:22:35,780 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:35,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T15:22:35,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b3e4e8e12ef,33849,1730992951906 in 353 msec 2024-11-07T15:22:35,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T15:22:35,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 675 msec 2024-11-07T15:22:35,799 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T15:22:35,799 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-07T15:22:35,820 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:22:35,821 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:22:35,844 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:22:35,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43273, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:22:35,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7830 sec 2024-11-07T15:22:35,876 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1730992955875, completionTime=-1 2024-11-07T15:22:35,879 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-07T15:22:35,879 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-07T15:22:35,915 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-07T15:22:35,915 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730993015915 2024-11-07T15:22:35,915 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730993075915 2024-11-07T15:22:35,915 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 35 msec 2024-11-07T15:22:35,918 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b3e4e8e12ef,46811,1730992951165-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:35,918 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b3e4e8e12ef,46811,1730992951165-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:35,918 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b3e4e8e12ef,46811,1730992951165-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:35,920 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7b3e4e8e12ef:46811, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:35,920 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:35,921 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:35,927 DEBUG [master/7b3e4e8e12ef:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-07T15:22:35,952 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.760sec 2024-11-07T15:22:35,953 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T15:22:35,955 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T15:22:35,956 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T15:22:35,957 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T15:22:35,957 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T15:22:35,958 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b3e4e8e12ef,46811,1730992951165-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T15:22:35,959 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b3e4e8e12ef,46811,1730992951165-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T15:22:35,969 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-07T15:22:35,970 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T15:22:35,970 INFO [master/7b3e4e8e12ef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b3e4e8e12ef,46811,1730992951165-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T15:22:36,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70868112, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:36,034 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-07T15:22:36,034 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-07T15:22:36,037 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:22:36,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:22:36,046 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:22:36,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:22:36,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:22:36,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c9a360, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:36,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:22:36,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:22:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:36,055 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:22:36,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@605dd515, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:36,058 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:22:36,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:22:36,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:22:36,067 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:22:36,090 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7b3e4e8e12ef,46811,1730992951165 2024-11-07T15:22:36,101 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=389, ProcessCount=11, AvailableMemoryMB=2871 2024-11-07T15:22:36,109 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T15:22:36,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b3e4e8e12ef,46811,1730992951165 2024-11-07T15:22:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1109f50e 2024-11-07T15:22:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T15:22:36,119 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T15:22:36,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:22:36,145 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:22:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T15:22:36,161 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T15:22:36,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-07T15:22:36,164 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:36,168 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T15:22:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T15:22:36,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741835_1011 (size=959) 2024-11-07T15:22:36,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T15:22:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T15:22:36,590 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938 2024-11-07T15:22:36,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741836_1012 (size=53) 2024-11-07T15:22:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T15:22:37,000 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:22:37,000 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 86b0da056c7c770742d9ba46450cf601, disabling compactions & flushes 2024-11-07T15:22:37,000 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:37,000 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:37,000 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. after waiting 0 ms 2024-11-07T15:22:37,000 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:37,000 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:37,000 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 86b0da056c7c770742d9ba46450cf601: Waiting for close lock at 1730992957000Disabling compacts and flushes for region at 1730992957000Disabling writes for close at 1730992957000Writing region close event to WAL at 1730992957000Closed at 1730992957000 2024-11-07T15:22:37,003 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T15:22:37,007 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730992957003"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730992957003"}]},"ts":"1730992957003"} 2024-11-07T15:22:37,013 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T15:22:37,015 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T15:22:37,017 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730992957015"}]},"ts":"1730992957015"} 2024-11-07T15:22:37,022 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T15:22:37,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86b0da056c7c770742d9ba46450cf601, ASSIGN}] 2024-11-07T15:22:37,026 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86b0da056c7c770742d9ba46450cf601, ASSIGN 2024-11-07T15:22:37,028 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86b0da056c7c770742d9ba46450cf601, ASSIGN; state=OFFLINE, location=7b3e4e8e12ef,33849,1730992951906; forceNewPlan=false, retain=false 2024-11-07T15:22:37,180 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=86b0da056c7c770742d9ba46450cf601, regionState=OPENING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:37,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86b0da056c7c770742d9ba46450cf601, ASSIGN because future has completed 2024-11-07T15:22:37,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:22:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T15:22:37,350 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:37,350 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:22:37,350 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:37,350 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:22:37,351 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:37,351 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:37,353 INFO [StoreOpener-86b0da056c7c770742d9ba46450cf601-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:37,356 INFO [StoreOpener-86b0da056c7c770742d9ba46450cf601-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86b0da056c7c770742d9ba46450cf601 columnFamilyName A 2024-11-07T15:22:37,356 DEBUG [StoreOpener-86b0da056c7c770742d9ba46450cf601-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:37,357 INFO [StoreOpener-86b0da056c7c770742d9ba46450cf601-1 {}] regionserver.HStore(327): Store=86b0da056c7c770742d9ba46450cf601/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:22:37,357 INFO [StoreOpener-86b0da056c7c770742d9ba46450cf601-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:37,360 INFO [StoreOpener-86b0da056c7c770742d9ba46450cf601-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86b0da056c7c770742d9ba46450cf601 columnFamilyName B 2024-11-07T15:22:37,360 DEBUG [StoreOpener-86b0da056c7c770742d9ba46450cf601-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:37,361 INFO [StoreOpener-86b0da056c7c770742d9ba46450cf601-1 {}] regionserver.HStore(327): Store=86b0da056c7c770742d9ba46450cf601/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:22:37,361 INFO [StoreOpener-86b0da056c7c770742d9ba46450cf601-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:37,364 INFO [StoreOpener-86b0da056c7c770742d9ba46450cf601-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86b0da056c7c770742d9ba46450cf601 columnFamilyName C 2024-11-07T15:22:37,364 DEBUG [StoreOpener-86b0da056c7c770742d9ba46450cf601-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:22:37,365 INFO [StoreOpener-86b0da056c7c770742d9ba46450cf601-1 {}] regionserver.HStore(327): Store=86b0da056c7c770742d9ba46450cf601/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:22:37,366 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:37,368 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:37,369 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:37,371 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:37,371 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:37,372 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:22:37,375 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:37,380 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T15:22:37,382 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 86b0da056c7c770742d9ba46450cf601; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64746868, jitterRate=-0.03519648313522339}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:22:37,382 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:37,384 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 86b0da056c7c770742d9ba46450cf601: Running coprocessor pre-open hook at 1730992957351Writing region info on filesystem at 1730992957351Initializing all the Stores at 1730992957353 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730992957353Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730992957353Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730992957353Cleaning up temporary data from old regions at 1730992957371 (+18 ms)Running coprocessor post-open hooks at 1730992957382 (+11 ms)Region opened successfully at 1730992957383 (+1 ms) 2024-11-07T15:22:37,387 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., pid=6, masterSystemTime=1730992957342 2024-11-07T15:22:37,392 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:37,392 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:37,394 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=86b0da056c7c770742d9ba46450cf601, regionState=OPEN, openSeqNum=2, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:37,399 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46811 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=7b3e4e8e12ef,33849,1730992951906, table=TestAcidGuarantees, region=86b0da056c7c770742d9ba46450cf601. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T15:22:37,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:22:37,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-07T15:22:37,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 in 218 msec 2024-11-07T15:22:37,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-07T15:22:37,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86b0da056c7c770742d9ba46450cf601, ASSIGN in 387 msec 2024-11-07T15:22:37,422 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T15:22:37,423 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730992957422"}]},"ts":"1730992957422"} 2024-11-07T15:22:37,427 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T15:22:37,429 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T15:22:37,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2800 sec 2024-11-07T15:22:38,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T15:22:38,322 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:22:38,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@694aa0a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,327 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:22:38,327 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:22:38,328 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:22:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:22:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:22:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d417a52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:22:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:22:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:38,332 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:22:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9a777e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,334 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:22:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:22:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:22:38,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:22:38,342 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T15:22:38,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b3e4e8e12ef,46811,1730992951165 2024-11-07T15:22:38,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@144e408 2024-11-07T15:22:38,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T15:22:38,350 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T15:22:38,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e67cef8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,354 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:22:38,354 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:22:38,356 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:22:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:22:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:22:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175526a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:22:38,358 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:22:38,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:38,360 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:22:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af2c0a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76916b8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,363 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:22:38,363 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:22:38,365 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:22:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:22:38,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:22:38,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ebcbe76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:22:38,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:22:38,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:38,370 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:22:38,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1612488f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e786f85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,374 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:22:38,374 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:22:38,376 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:22:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:22:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:22:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:22:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:22:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:38,380 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:22:38,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc4f8ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eea3ba9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:22:38,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:22:38,384 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:22:38,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:22:38,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:22:38,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2555cfdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:22:38,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:22:38,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:38,388 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:22:38,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58577299, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d6a39a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,391 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:22:38,391 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:22:38,392 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:22:38,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:22:38,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:22:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70f9f32c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:22:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:22:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:38,396 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:22:38,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff3582c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7371bc35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:22:38,399 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:22:38,400 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:22:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:22:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:22:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2521eed1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:22:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:22:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:38,403 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:22:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@359c1e82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9cd7d7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:22:38,406 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:22:38,407 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:22:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:22:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:22:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44bee5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:22:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:22:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:38,409 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:22:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a98f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62df2686, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,412 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:22:38,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:22:38,414 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:22:38,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:22:38,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:22:38,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f232f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:22:38,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:22:38,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:38,417 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:22:38,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be3f202, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@630fc7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:22:38,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:22:38,421 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:22:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:22:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:22:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f437cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:22:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:22:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:38,424 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:22:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f72cf73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:22:38,428 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:22:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:22:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:22:38,432 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:22:38,432 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:22:38,433 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:22:38,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:22:38,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:22:38,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:22:38,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:22:38,435 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:22:38,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:22:38,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:22:38,436 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:22:38,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:22:38,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:22:38,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:22:38,443 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:22:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:22:38,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:22:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:22:38,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:22:38,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:22:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:22:38,452 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:22:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:22:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:22:38,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:22:38,456 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:22:38,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:22:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:22:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:22:38,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:22:38,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-07T15:22:38,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:22:38,466 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:22:38,469 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:22:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T15:22:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:22:38,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:22:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:22:38,474 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:22:38,475 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:22:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:22:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:22:38,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:22:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:22:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:22:38,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:22:38,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:22:38,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:22:38,536 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:22:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:38,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:22:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T15:22:38,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T15:22:38,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:38,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:38,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/3dd1233493b44b47a3500dbc9bae41ea is 50, key is test_row_0/A:col10/1730992958541/Put/seqid=0 2024-11-07T15:22:38,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:38,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:38,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:38,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:38,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741837_1013 (size=14341) 2024-11-07T15:22:38,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/3dd1233493b44b47a3500dbc9bae41ea 2024-11-07T15:22:38,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993018712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993018719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:38,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993018720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:38,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:38,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993018727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:38,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993018735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T15:22:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:38,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/20f448f38e4641e8bff89ef6bf7f8962 is 50, key is test_row_0/B:col10/1730992958541/Put/seqid=0 2024-11-07T15:22:38,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T15:22:38,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741838_1014 (size=12001) 2024-11-07T15:22:38,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:38,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:38,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:38,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:38,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:38,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:38,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993018912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:38,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:38,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:38,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993018913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:38,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:38,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993018914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:38,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:38,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:38,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993018915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:38,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993018914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:39,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T15:22:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:39,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T15:22:39,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993019122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:39,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:39,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993019132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993019133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:39,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993019134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:39,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:39,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993019134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:39,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:39,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T15:22:39,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:39,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:39,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:39,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/20f448f38e4641e8bff89ef6bf7f8962 2024-11-07T15:22:39,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/b9b3502525f6446db9015e8a19a5a6f3 is 50, key is test_row_0/C:col10/1730992958541/Put/seqid=0 2024-11-07T15:22:39,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T15:22:39,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:39,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:39,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:39,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741839_1015 (size=12001) 2024-11-07T15:22:39,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993019434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:39,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:39,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993019441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:39,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993019451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:39,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993019452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:39,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993019453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:39,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T15:22:39,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:39,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:39,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:39,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T15:22:39,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T15:22:39,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:39,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:39,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:39,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/b9b3502525f6446db9015e8a19a5a6f3 2024-11-07T15:22:39,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/3dd1233493b44b47a3500dbc9bae41ea as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/3dd1233493b44b47a3500dbc9bae41ea 2024-11-07T15:22:39,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/3dd1233493b44b47a3500dbc9bae41ea, entries=200, sequenceid=14, filesize=14.0 K 2024-11-07T15:22:39,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/20f448f38e4641e8bff89ef6bf7f8962 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/20f448f38e4641e8bff89ef6bf7f8962 2024-11-07T15:22:39,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T15:22:39,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:39,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:39,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:39,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:39,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/20f448f38e4641e8bff89ef6bf7f8962, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T15:22:39,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/b9b3502525f6446db9015e8a19a5a6f3 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b9b3502525f6446db9015e8a19a5a6f3 2024-11-07T15:22:39,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b9b3502525f6446db9015e8a19a5a6f3, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T15:22:39,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 86b0da056c7c770742d9ba46450cf601 in 1348ms, sequenceid=14, compaction requested=false 2024-11-07T15:22:39,894 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T15:22:39,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:39,953 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:22:39,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T15:22:39,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/226d5ce0bb0e4669b4d064e7252e890a is 50, key is test_row_0/A:col10/1730992959950/Put/seqid=0 2024-11-07T15:22:40,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741840_1016 (size=14341) 2024-11-07T15:22:40,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T15:22:40,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:40,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:40,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:40,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:40,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:40,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993019995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/226d5ce0bb0e4669b4d064e7252e890a 2024-11-07T15:22:40,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993019998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993020001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993020005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993020008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/28908b548ef64b9da9fff0f7d9074ea7 is 50, key is test_row_0/B:col10/1730992959950/Put/seqid=0 2024-11-07T15:22:40,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741841_1017 (size=12001) 2024-11-07T15:22:40,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/28908b548ef64b9da9fff0f7d9074ea7 2024-11-07T15:22:40,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/b27cf54d98ef4d9d8cfc5a69c9ef676f is 50, key is test_row_0/C:col10/1730992959950/Put/seqid=0 2024-11-07T15:22:40,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993020122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993020123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993020131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993020132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993020142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T15:22:40,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:40,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:40,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:40,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:40,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:40,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741842_1018 (size=12001) 2024-11-07T15:22:40,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/b27cf54d98ef4d9d8cfc5a69c9ef676f 2024-11-07T15:22:40,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/226d5ce0bb0e4669b4d064e7252e890a as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/226d5ce0bb0e4669b4d064e7252e890a 2024-11-07T15:22:40,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/226d5ce0bb0e4669b4d064e7252e890a, entries=200, sequenceid=38, filesize=14.0 K 2024-11-07T15:22:40,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/28908b548ef64b9da9fff0f7d9074ea7 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/28908b548ef64b9da9fff0f7d9074ea7 2024-11-07T15:22:40,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/28908b548ef64b9da9fff0f7d9074ea7, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T15:22:40,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/b27cf54d98ef4d9d8cfc5a69c9ef676f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b27cf54d98ef4d9d8cfc5a69c9ef676f 2024-11-07T15:22:40,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b27cf54d98ef4d9d8cfc5a69c9ef676f, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T15:22:40,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 86b0da056c7c770742d9ba46450cf601 in 324ms, sequenceid=38, compaction requested=false 2024-11-07T15:22:40,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:40,293 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-07T15:22:40,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T15:22:40,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:40,323 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T15:22:40,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/bf17493135a047e0b6513fea6c750483 is 50, key is test_row_0/A:col10/1730992959963/Put/seqid=0 2024-11-07T15:22:40,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:40,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741843_1019 (size=12001) 2024-11-07T15:22:40,397 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/bf17493135a047e0b6513fea6c750483 2024-11-07T15:22:40,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/dbfeeaa9967e4e58aa6f1525b2118cb8 is 50, key is test_row_0/B:col10/1730992959963/Put/seqid=0 2024-11-07T15:22:40,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741844_1020 (size=12001) 2024-11-07T15:22:40,464 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/dbfeeaa9967e4e58aa6f1525b2118cb8 2024-11-07T15:22:40,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993020456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993020459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993020465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/4f56cb5d1eff41b89a5d413092c9f3c9 is 50, key is test_row_0/C:col10/1730992959963/Put/seqid=0 2024-11-07T15:22:40,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993020469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993020474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741845_1021 (size=12001) 2024-11-07T15:22:40,528 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/4f56cb5d1eff41b89a5d413092c9f3c9 2024-11-07T15:22:40,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/bf17493135a047e0b6513fea6c750483 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/bf17493135a047e0b6513fea6c750483 2024-11-07T15:22:40,561 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/bf17493135a047e0b6513fea6c750483, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T15:22:40,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/dbfeeaa9967e4e58aa6f1525b2118cb8 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/dbfeeaa9967e4e58aa6f1525b2118cb8 2024-11-07T15:22:40,590 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/dbfeeaa9967e4e58aa6f1525b2118cb8, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T15:22:40,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993020582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/4f56cb5d1eff41b89a5d413092c9f3c9 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/4f56cb5d1eff41b89a5d413092c9f3c9 2024-11-07T15:22:40,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993020582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993020591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993020601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993020602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,615 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/4f56cb5d1eff41b89a5d413092c9f3c9, entries=150, sequenceid=50, filesize=11.7 K 2024-11-07T15:22:40,617 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 86b0da056c7c770742d9ba46450cf601 in 294ms, sequenceid=50, compaction requested=true 2024-11-07T15:22:40,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:40,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:40,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-07T15:22:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-07T15:22:40,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-07T15:22:40,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1500 sec 2024-11-07T15:22:40,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.1840 sec 2024-11-07T15:22:40,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T15:22:40,633 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:22:40,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:22:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-07T15:22:40,640 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:22:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T15:22:40,642 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:22:40,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:22:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T15:22:40,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T15:22:40,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:40,802 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T15:22:40,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:40,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/828dc33b7b0448e0bb977d09b1c73af7 is 50, key is test_row_0/A:col10/1730992960469/Put/seqid=0 2024-11-07T15:22:40,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741846_1022 (size=12001) 2024-11-07T15:22:40,845 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/828dc33b7b0448e0bb977d09b1c73af7 2024-11-07T15:22:40,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993020837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993020847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993020848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993020848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993020855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/8bb9f41c0c7940a0922dc1783537e3f2 is 50, key is test_row_0/B:col10/1730992960469/Put/seqid=0 2024-11-07T15:22:40,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741847_1023 (size=12001) 2024-11-07T15:22:40,903 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/8bb9f41c0c7940a0922dc1783537e3f2 2024-11-07T15:22:40,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/55e4d5939b504b649c4584e2a0fc2e36 is 50, key is test_row_0/C:col10/1730992960469/Put/seqid=0 2024-11-07T15:22:40,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T15:22:40,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741848_1024 (size=12001) 2024-11-07T15:22:40,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993020961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993020961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,970 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/55e4d5939b504b649c4584e2a0fc2e36 2024-11-07T15:22:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993020971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993020971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993020972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:40,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:40,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:40,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/828dc33b7b0448e0bb977d09b1c73af7 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/828dc33b7b0448e0bb977d09b1c73af7 2024-11-07T15:22:41,012 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/828dc33b7b0448e0bb977d09b1c73af7, entries=150, sequenceid=74, filesize=11.7 K 2024-11-07T15:22:41,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/8bb9f41c0c7940a0922dc1783537e3f2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8bb9f41c0c7940a0922dc1783537e3f2 2024-11-07T15:22:41,047 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8bb9f41c0c7940a0922dc1783537e3f2, entries=150, sequenceid=74, filesize=11.7 K 2024-11-07T15:22:41,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/55e4d5939b504b649c4584e2a0fc2e36 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/55e4d5939b504b649c4584e2a0fc2e36 2024-11-07T15:22:41,076 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/55e4d5939b504b649c4584e2a0fc2e36, entries=150, sequenceid=74, filesize=11.7 K 2024-11-07T15:22:41,079 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 86b0da056c7c770742d9ba46450cf601 in 277ms, sequenceid=74, compaction requested=true 2024-11-07T15:22:41,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:41,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:41,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-07T15:22:41,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-07T15:22:41,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-07T15:22:41,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 443 msec 2024-11-07T15:22:41,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 458 msec 2024-11-07T15:22:41,178 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T15:22:41,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:41,189 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:22:41,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T15:22:41,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/60231290e67345d491a1c5efbacee287 is 50, key is test_row_0/A:col10/1730992960841/Put/seqid=0 2024-11-07T15:22:41,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741849_1025 (size=14341) 2024-11-07T15:22:41,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/60231290e67345d491a1c5efbacee287 2024-11-07T15:22:41,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T15:22:41,271 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:22:41,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:22:41,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-07T15:22:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T15:22:41,276 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:22:41,278 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:22:41,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:22:41,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/54f1f3025a194b51a1da46ae7519c341 is 50, key is test_row_0/B:col10/1730992960841/Put/seqid=0 2024-11-07T15:22:41,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741850_1026 (size=12001) 2024-11-07T15:22:41,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993021323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:41,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:41,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993021323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:41,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:41,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993021323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:41,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:41,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993021331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:41,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:41,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993021333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:41,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T15:22:41,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T15:22:41,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:41,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:41,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:41,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:41,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:41,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993021441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:41,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:41,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993021442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993021442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:41,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993021442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:41,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993021451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:41,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:41,496 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T15:22:41,497 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-07T15:22:41,500 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T15:22:41,500 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-07T15:22:41,501 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-07T15:22:41,501 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-07T15:22:41,502 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T15:22:41,502 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-07T15:22:41,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T15:22:41,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:41,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:41,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T15:22:41,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:41,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:41,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:41,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993021651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:41,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:41,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993021651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:41,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:41,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:41,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:41,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993021661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:41,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:41,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:41,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:41,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993021661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:41,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993021661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:41,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:41,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:41,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/54f1f3025a194b51a1da46ae7519c341 2024-11-07T15:22:41,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T15:22:41,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:41,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:41,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:41,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:41,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:41,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/172f47523ebd4495bfed6cf46e4520e3 is 50, key is test_row_0/C:col10/1730992960841/Put/seqid=0 2024-11-07T15:22:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:41,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741851_1027 (size=12001) 2024-11-07T15:22:41,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/172f47523ebd4495bfed6cf46e4520e3 2024-11-07T15:22:41,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/60231290e67345d491a1c5efbacee287 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/60231290e67345d491a1c5efbacee287 2024-11-07T15:22:41,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/60231290e67345d491a1c5efbacee287, entries=200, sequenceid=88, filesize=14.0 K 2024-11-07T15:22:41,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/54f1f3025a194b51a1da46ae7519c341 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/54f1f3025a194b51a1da46ae7519c341 2024-11-07T15:22:41,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/54f1f3025a194b51a1da46ae7519c341, entries=150, sequenceid=88, filesize=11.7 K 2024-11-07T15:22:41,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/172f47523ebd4495bfed6cf46e4520e3 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/172f47523ebd4495bfed6cf46e4520e3 2024-11-07T15:22:41,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/172f47523ebd4495bfed6cf46e4520e3, entries=150, sequenceid=88, filesize=11.7 K 2024-11-07T15:22:41,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 86b0da056c7c770742d9ba46450cf601 in 677ms, sequenceid=88, compaction requested=true 2024-11-07T15:22:41,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:41,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:22:41,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:41,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:22:41,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:41,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:22:41,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:41,882 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T15:22:41,883 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T15:22:41,887 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T15:22:41,892 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/B is initiating minor compaction (all files) 2024-11-07T15:22:41,892 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/B in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:41,893 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/20f448f38e4641e8bff89ef6bf7f8962, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/28908b548ef64b9da9fff0f7d9074ea7, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/dbfeeaa9967e4e58aa6f1525b2118cb8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8bb9f41c0c7940a0922dc1783537e3f2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/54f1f3025a194b51a1da46ae7519c341] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=58.6 K 2024-11-07T15:22:41,897 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 20f448f38e4641e8bff89ef6bf7f8962, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730992958526 2024-11-07T15:22:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T15:22:41,902 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 28908b548ef64b9da9fff0f7d9074ea7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730992958622 2024-11-07T15:22:41,902 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 67025 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T15:22:41,902 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/A is initiating minor compaction (all files) 2024-11-07T15:22:41,902 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/A in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:41,902 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/3dd1233493b44b47a3500dbc9bae41ea, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/226d5ce0bb0e4669b4d064e7252e890a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/bf17493135a047e0b6513fea6c750483, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/828dc33b7b0448e0bb977d09b1c73af7, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/60231290e67345d491a1c5efbacee287] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=65.5 K 2024-11-07T15:22:41,903 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting dbfeeaa9967e4e58aa6f1525b2118cb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730992959963 2024-11-07T15:22:41,903 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dd1233493b44b47a3500dbc9bae41ea, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730992958513 2024-11-07T15:22:41,904 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bb9f41c0c7940a0922dc1783537e3f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1730992960456 2024-11-07T15:22:41,905 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 54f1f3025a194b51a1da46ae7519c341, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1730992960836 2024-11-07T15:22:41,905 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 226d5ce0bb0e4669b4d064e7252e890a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730992958622 2024-11-07T15:22:41,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T15:22:41,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:41,906 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T15:22:41,906 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf17493135a047e0b6513fea6c750483, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730992959963 2024-11-07T15:22:41,907 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 828dc33b7b0448e0bb977d09b1c73af7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1730992960456 2024-11-07T15:22:41,910 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60231290e67345d491a1c5efbacee287, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1730992960836 2024-11-07T15:22:41,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/6cfe8abd1fb544df85a9de4b4428ee86 is 50, key is test_row_0/A:col10/1730992961319/Put/seqid=0 2024-11-07T15:22:41,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741852_1028 (size=12001) 2024-11-07T15:22:41,950 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/6cfe8abd1fb544df85a9de4b4428ee86 2024-11-07T15:22:41,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:41,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/93b5fba9e47d4f7ab5799a9d27beeb12 is 50, key is test_row_0/B:col10/1730992961319/Put/seqid=0 2024-11-07T15:22:42,022 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#B#compaction#17 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:42,023 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#A#compaction#16 average throughput is 0.20 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:42,024 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/c53125f50c20465eb616a49f93195cb5 is 50, key is test_row_0/A:col10/1730992960841/Put/seqid=0 2024-11-07T15:22:42,028 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/92b1edcba0a14ab59f13156f79720a76 is 50, key is test_row_0/B:col10/1730992960841/Put/seqid=0 2024-11-07T15:22:42,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741853_1029 (size=12001) 2024-11-07T15:22:42,039 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/93b5fba9e47d4f7ab5799a9d27beeb12 2024-11-07T15:22:42,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993022033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993022033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993022058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993022060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993022055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741854_1030 (size=12173) 2024-11-07T15:22:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/5db684d7ea4847938d6d8d9edd79d057 is 50, key is test_row_0/C:col10/1730992961319/Put/seqid=0 2024-11-07T15:22:42,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741855_1031 (size=12173) 2024-11-07T15:22:42,125 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/c53125f50c20465eb616a49f93195cb5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/c53125f50c20465eb616a49f93195cb5 2024-11-07T15:22:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,157 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/92b1edcba0a14ab59f13156f79720a76 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/92b1edcba0a14ab59f13156f79720a76 2024-11-07T15:22:42,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993022172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993022172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,179 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/B of 86b0da056c7c770742d9ba46450cf601 into 92b1edcba0a14ab59f13156f79720a76(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:42,179 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:42,180 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/A of 86b0da056c7c770742d9ba46450cf601 into c53125f50c20465eb616a49f93195cb5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:42,180 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:42,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741856_1032 (size=12001) 2024-11-07T15:22:42,186 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/B, priority=11, startTime=1730992961879; duration=0sec 2024-11-07T15:22:42,187 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/A, priority=11, startTime=1730992961868; duration=0sec 2024-11-07T15:22:42,187 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:42,187 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:B 2024-11-07T15:22:42,187 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:42,187 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:A 2024-11-07T15:22:42,188 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T15:22:42,192 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T15:22:42,192 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/C is initiating minor compaction (all files) 2024-11-07T15:22:42,192 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/C in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:42,193 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b9b3502525f6446db9015e8a19a5a6f3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b27cf54d98ef4d9d8cfc5a69c9ef676f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/4f56cb5d1eff41b89a5d413092c9f3c9, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/55e4d5939b504b649c4584e2a0fc2e36, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/172f47523ebd4495bfed6cf46e4520e3] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=58.6 K 2024-11-07T15:22:42,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993022181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,196 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9b3502525f6446db9015e8a19a5a6f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730992958526 2024-11-07T15:22:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,199 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting b27cf54d98ef4d9d8cfc5a69c9ef676f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730992958622 2024-11-07T15:22:42,200 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f56cb5d1eff41b89a5d413092c9f3c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1730992959963 2024-11-07T15:22:42,202 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55e4d5939b504b649c4584e2a0fc2e36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1730992960456 2024-11-07T15:22:42,203 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 172f47523ebd4495bfed6cf46e4520e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1730992960836 2024-11-07T15:22:42,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993022231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993022231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,254 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#C#compaction#20 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:42,255 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/615604929ad04eeca86e5c8ca00ff195 is 50, key is test_row_0/C:col10/1730992960841/Put/seqid=0 2024-11-07T15:22:42,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741857_1033 (size=12173) 2024-11-07T15:22:42,318 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/615604929ad04eeca86e5c8ca00ff195 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/615604929ad04eeca86e5c8ca00ff195 2024-11-07T15:22:42,334 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/C of 86b0da056c7c770742d9ba46450cf601 into 615604929ad04eeca86e5c8ca00ff195(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:42,334 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:42,334 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/C, priority=11, startTime=1730992961879; duration=0sec 2024-11-07T15:22:42,334 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:42,334 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:C 2024-11-07T15:22:42,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993022381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993022381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993022402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T15:22:42,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993022441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993022452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,588 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/5db684d7ea4847938d6d8d9edd79d057 2024-11-07T15:22:42,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/6cfe8abd1fb544df85a9de4b4428ee86 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/6cfe8abd1fb544df85a9de4b4428ee86 2024-11-07T15:22:42,614 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/6cfe8abd1fb544df85a9de4b4428ee86, entries=150, sequenceid=110, filesize=11.7 K 2024-11-07T15:22:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/93b5fba9e47d4f7ab5799a9d27beeb12 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/93b5fba9e47d4f7ab5799a9d27beeb12 2024-11-07T15:22:42,629 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/93b5fba9e47d4f7ab5799a9d27beeb12, entries=150, sequenceid=110, filesize=11.7 K 2024-11-07T15:22:42,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/5db684d7ea4847938d6d8d9edd79d057 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/5db684d7ea4847938d6d8d9edd79d057 2024-11-07T15:22:42,676 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/5db684d7ea4847938d6d8d9edd79d057, entries=150, sequenceid=110, filesize=11.7 K 2024-11-07T15:22:42,679 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 86b0da056c7c770742d9ba46450cf601 in 774ms, sequenceid=110, compaction requested=false 2024-11-07T15:22:42,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:42,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:42,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-07T15:22:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-07T15:22:42,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-07T15:22:42,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-11-07T15:22:42,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.4210 sec 2024-11-07T15:22:42,701 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:22:42,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T15:22:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:42,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/96fa6f19f383416587bcd180098e6dbf is 50, key is test_row_0/A:col10/1730992962049/Put/seqid=0 2024-11-07T15:22:42,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741858_1034 (size=12001) 2024-11-07T15:22:42,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993022824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993022824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993022831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993022831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993022831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993022942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993022942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993022942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993022951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993022951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:42,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993023151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993023151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993023151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993023161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993023161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/96fa6f19f383416587bcd180098e6dbf 2024-11-07T15:22:43,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/bb39c73020d54ab89e1a8b8ffb333ad4 is 50, key is test_row_0/B:col10/1730992962049/Put/seqid=0 2024-11-07T15:22:43,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741859_1035 (size=12001) 2024-11-07T15:22:43,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/bb39c73020d54ab89e1a8b8ffb333ad4 2024-11-07T15:22:43,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/71e26f5da3bd4f12894132735853b131 is 50, key is test_row_0/C:col10/1730992962049/Put/seqid=0 2024-11-07T15:22:43,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741860_1036 (size=12001) 2024-11-07T15:22:43,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/71e26f5da3bd4f12894132735853b131 2024-11-07T15:22:43,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/96fa6f19f383416587bcd180098e6dbf as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/96fa6f19f383416587bcd180098e6dbf 2024-11-07T15:22:43,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/96fa6f19f383416587bcd180098e6dbf, entries=150, sequenceid=128, filesize=11.7 K 2024-11-07T15:22:43,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/bb39c73020d54ab89e1a8b8ffb333ad4 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/bb39c73020d54ab89e1a8b8ffb333ad4 2024-11-07T15:22:43,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/bb39c73020d54ab89e1a8b8ffb333ad4, entries=150, sequenceid=128, filesize=11.7 K 2024-11-07T15:22:43,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/71e26f5da3bd4f12894132735853b131 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/71e26f5da3bd4f12894132735853b131 2024-11-07T15:22:43,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/71e26f5da3bd4f12894132735853b131, entries=150, sequenceid=128, filesize=11.7 K 2024-11-07T15:22:43,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=134.18 KB/137400 for 86b0da056c7c770742d9ba46450cf601 in 618ms, sequenceid=128, compaction requested=true 2024-11-07T15:22:43,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:43,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:22:43,320 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:22:43,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:43,320 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:22:43,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:22:43,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:43,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:22:43,323 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:22:43,322 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:22:43,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:43,323 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/A is initiating minor compaction (all files) 2024-11-07T15:22:43,323 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/B is initiating minor compaction (all files) 2024-11-07T15:22:43,323 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/B in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:43,323 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/A in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:43,324 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/c53125f50c20465eb616a49f93195cb5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/6cfe8abd1fb544df85a9de4b4428ee86, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/96fa6f19f383416587bcd180098e6dbf] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=35.3 K 2024-11-07T15:22:43,324 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/92b1edcba0a14ab59f13156f79720a76, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/93b5fba9e47d4f7ab5799a9d27beeb12, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/bb39c73020d54ab89e1a8b8ffb333ad4] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=35.3 K 2024-11-07T15:22:43,325 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting c53125f50c20465eb616a49f93195cb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1730992960836 2024-11-07T15:22:43,326 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 92b1edcba0a14ab59f13156f79720a76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1730992960836 2024-11-07T15:22:43,327 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cfe8abd1fb544df85a9de4b4428ee86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1730992961319 2024-11-07T15:22:43,327 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 93b5fba9e47d4f7ab5799a9d27beeb12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1730992961319 2024-11-07T15:22:43,328 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting bb39c73020d54ab89e1a8b8ffb333ad4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1730992962049 2024-11-07T15:22:43,329 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96fa6f19f383416587bcd180098e6dbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1730992962049 2024-11-07T15:22:43,363 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#A#compaction#24 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:43,364 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/9e2b91b62bc44198a44c59455975b6b1 is 50, key is test_row_0/A:col10/1730992962049/Put/seqid=0 2024-11-07T15:22:43,365 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#B#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:43,366 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/a3d8d90f1d5c41bba0823d562be28524 is 50, key is test_row_0/B:col10/1730992962049/Put/seqid=0 2024-11-07T15:22:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T15:22:43,422 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:22:43,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:22:43,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-07T15:22:43,426 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:22:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T15:22:43,427 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:22:43,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:22:43,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741862_1038 (size=12275) 2024-11-07T15:22:43,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741861_1037 (size=12275) 2024-11-07T15:22:43,454 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/a3d8d90f1d5c41bba0823d562be28524 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/a3d8d90f1d5c41bba0823d562be28524 2024-11-07T15:22:43,469 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:22:43,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T15:22:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:43,474 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/B of 86b0da056c7c770742d9ba46450cf601 into a3d8d90f1d5c41bba0823d562be28524(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:43,474 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:43,475 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/B, priority=13, startTime=1730992963320; duration=0sec 2024-11-07T15:22:43,475 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:43,475 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:B 2024-11-07T15:22:43,475 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:22:43,480 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:22:43,480 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/C is initiating minor compaction (all files) 2024-11-07T15:22:43,480 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/C in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:43,480 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/615604929ad04eeca86e5c8ca00ff195, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/5db684d7ea4847938d6d8d9edd79d057, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/71e26f5da3bd4f12894132735853b131] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=35.3 K 2024-11-07T15:22:43,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/3faf34ac2f364e4b8f8e1bad44c7f4bc is 50, key is test_row_0/A:col10/1730992963466/Put/seqid=0 2024-11-07T15:22:43,482 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 615604929ad04eeca86e5c8ca00ff195, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1730992960836 2024-11-07T15:22:43,483 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 5db684d7ea4847938d6d8d9edd79d057, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1730992961319 2024-11-07T15:22:43,485 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 71e26f5da3bd4f12894132735853b131, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1730992962049 2024-11-07T15:22:43,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993023495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993023499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993023499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,522 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#C#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:43,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,523 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/6d42596a070c4b3596071655809861cf is 50, key is test_row_0/C:col10/1730992962049/Put/seqid=0 2024-11-07T15:22:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993023506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993023492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T15:22:43,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741863_1039 (size=14541) 2024-11-07T15:22:43,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741864_1040 (size=12275) 2024-11-07T15:22:43,575 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/6d42596a070c4b3596071655809861cf as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/6d42596a070c4b3596071655809861cf 2024-11-07T15:22:43,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T15:22:43,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:43,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:43,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:43,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:43,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:43,590 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/C of 86b0da056c7c770742d9ba46450cf601 into 6d42596a070c4b3596071655809861cf(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:43,590 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:43,590 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/C, priority=13, startTime=1730992963321; duration=0sec 2024-11-07T15:22:43,591 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:43,591 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:C 2024-11-07T15:22:43,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993023611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993023611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993023612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993023631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993023631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T15:22:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:43,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:43,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:43,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T15:22:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:43,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993023822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993023822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993023822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993023841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993023841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:43,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:43,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:43,875 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/9e2b91b62bc44198a44c59455975b6b1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/9e2b91b62bc44198a44c59455975b6b1 2024-11-07T15:22:43,891 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/A of 86b0da056c7c770742d9ba46450cf601 into 9e2b91b62bc44198a44c59455975b6b1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:43,892 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:43,892 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/A, priority=13, startTime=1730992963320; duration=0sec 2024-11-07T15:22:43,892 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:43,892 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:A 2024-11-07T15:22:43,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T15:22:43,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:43,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:43,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:43,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:43,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:43,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/3faf34ac2f364e4b8f8e1bad44c7f4bc 2024-11-07T15:22:43,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/b0485c9ef95741d9a93ae6d73fa2aa93 is 50, key is test_row_0/B:col10/1730992963466/Put/seqid=0 2024-11-07T15:22:44,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741865_1041 (size=12151) 2024-11-07T15:22:44,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/b0485c9ef95741d9a93ae6d73fa2aa93 2024-11-07T15:22:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T15:22:44,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T15:22:44,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:44,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:44,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:44,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/1d856d1166664eb8b78525abf20ea95b is 50, key is test_row_0/C:col10/1730992963466/Put/seqid=0 2024-11-07T15:22:44,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741866_1042 (size=12151) 2024-11-07T15:22:44,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993024149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:44,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993024149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:44,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:44,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993024149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:44,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:44,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993024162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:44,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:44,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993024162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:44,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:44,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T15:22:44,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:44,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:44,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:44,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T15:22:44,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:44,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:44,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:44,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/1d856d1166664eb8b78525abf20ea95b 2024-11-07T15:22:44,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/3faf34ac2f364e4b8f8e1bad44c7f4bc as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/3faf34ac2f364e4b8f8e1bad44c7f4bc 2024-11-07T15:22:44,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T15:22:44,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:44,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:44,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:44,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/3faf34ac2f364e4b8f8e1bad44c7f4bc, entries=200, sequenceid=152, filesize=14.2 K 2024-11-07T15:22:44,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/b0485c9ef95741d9a93ae6d73fa2aa93 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b0485c9ef95741d9a93ae6d73fa2aa93 2024-11-07T15:22:44,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b0485c9ef95741d9a93ae6d73fa2aa93, entries=150, sequenceid=152, filesize=11.9 K 2024-11-07T15:22:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T15:22:44,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/1d856d1166664eb8b78525abf20ea95b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1d856d1166664eb8b78525abf20ea95b 2024-11-07T15:22:44,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1d856d1166664eb8b78525abf20ea95b, entries=150, sequenceid=152, filesize=11.9 K 2024-11-07T15:22:44,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 86b0da056c7c770742d9ba46450cf601 in 1123ms, sequenceid=152, compaction requested=false 2024-11-07T15:22:44,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:44,673 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:22:44,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T15:22:44,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T15:22:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:44,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/bec398fa2f93436cb820c9f537e9def6 is 50, key is test_row_0/A:col10/1730992964671/Put/seqid=0 2024-11-07T15:22:44,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741867_1043 (size=14541) 2024-11-07T15:22:44,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/bec398fa2f93436cb820c9f537e9def6 2024-11-07T15:22:44,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:44,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993024717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:44,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:44,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:44,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993024715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:44,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:44,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993024721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:44,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:44,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:44,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993024724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:44,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:44,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993024724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:44,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/7c53de08047e4d83b80a8a6c8b521996 is 50, key is test_row_0/B:col10/1730992964671/Put/seqid=0 2024-11-07T15:22:44,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741868_1044 (size=12151) 2024-11-07T15:22:44,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/7c53de08047e4d83b80a8a6c8b521996 2024-11-07T15:22:44,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/a34bb7b891534b868e86c1ff7b348a51 is 50, key is test_row_0/C:col10/1730992964671/Put/seqid=0 2024-11-07T15:22:44,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T15:22:44,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:44,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:44,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:44,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:44,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993024831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:44,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993024832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:44,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993024841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:44,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993024842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:44,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:44,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993024841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:44,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741869_1045 (size=12151) 2024-11-07T15:22:44,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/a34bb7b891534b868e86c1ff7b348a51 2024-11-07T15:22:44,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/bec398fa2f93436cb820c9f537e9def6 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/bec398fa2f93436cb820c9f537e9def6 2024-11-07T15:22:44,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/bec398fa2f93436cb820c9f537e9def6, entries=200, sequenceid=168, filesize=14.2 K 2024-11-07T15:22:44,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/7c53de08047e4d83b80a8a6c8b521996 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/7c53de08047e4d83b80a8a6c8b521996 2024-11-07T15:22:44,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/7c53de08047e4d83b80a8a6c8b521996, entries=150, sequenceid=168, filesize=11.9 K 2024-11-07T15:22:44,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/a34bb7b891534b868e86c1ff7b348a51 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/a34bb7b891534b868e86c1ff7b348a51 2024-11-07T15:22:44,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/a34bb7b891534b868e86c1ff7b348a51, entries=150, sequenceid=168, filesize=11.9 K 2024-11-07T15:22:44,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 86b0da056c7c770742d9ba46450cf601 in 247ms, sequenceid=168, compaction requested=true 2024-11-07T15:22:44,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:44,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:22:44,920 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:22:44,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:44,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:22:44,920 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:22:44,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:44,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:22:44,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:44,923 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41357 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:22:44,923 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:22:44,923 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/B is initiating minor compaction (all files) 2024-11-07T15:22:44,923 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/A is initiating minor compaction (all files) 2024-11-07T15:22:44,923 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/A in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:44,923 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/B in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:44,923 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/a3d8d90f1d5c41bba0823d562be28524, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b0485c9ef95741d9a93ae6d73fa2aa93, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/7c53de08047e4d83b80a8a6c8b521996] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=35.7 K 2024-11-07T15:22:44,923 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/9e2b91b62bc44198a44c59455975b6b1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/3faf34ac2f364e4b8f8e1bad44c7f4bc, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/bec398fa2f93436cb820c9f537e9def6] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=40.4 K 2024-11-07T15:22:44,924 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e2b91b62bc44198a44c59455975b6b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1730992962049 2024-11-07T15:22:44,924 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting a3d8d90f1d5c41bba0823d562be28524, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1730992962049 2024-11-07T15:22:44,925 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3faf34ac2f364e4b8f8e1bad44c7f4bc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1730992962815 2024-11-07T15:22:44,925 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting b0485c9ef95741d9a93ae6d73fa2aa93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1730992962815 2024-11-07T15:22:44,925 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c53de08047e4d83b80a8a6c8b521996, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730992963492 2024-11-07T15:22:44,926 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting bec398fa2f93436cb820c9f537e9def6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730992963492 2024-11-07T15:22:44,955 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#B#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:44,956 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/8879e2d5ca044cd0a9498bcc5e2db4fc is 50, key is test_row_0/B:col10/1730992964671/Put/seqid=0 2024-11-07T15:22:44,961 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#A#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:44,962 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/1341955a50384398bfd7d7e3333e7376 is 50, key is test_row_0/A:col10/1730992964671/Put/seqid=0 2024-11-07T15:22:44,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T15:22:44,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:44,992 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T15:22:45,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741870_1046 (size=12527) 2024-11-07T15:22:45,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741871_1047 (size=12527) 2024-11-07T15:22:45,011 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/8879e2d5ca044cd0a9498bcc5e2db4fc as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8879e2d5ca044cd0a9498bcc5e2db4fc 2024-11-07T15:22:45,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/7c7e5e5fe7be4ac3afec1cdaddb4fee8 is 50, key is test_row_0/A:col10/1730992964714/Put/seqid=0 2024-11-07T15:22:45,029 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/B of 86b0da056c7c770742d9ba46450cf601 into 8879e2d5ca044cd0a9498bcc5e2db4fc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:45,029 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:45,029 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/B, priority=13, startTime=1730992964920; duration=0sec 2024-11-07T15:22:45,030 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:45,030 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:B 2024-11-07T15:22:45,030 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:22:45,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741872_1048 (size=12151) 2024-11-07T15:22:45,035 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/7c7e5e5fe7be4ac3afec1cdaddb4fee8 2024-11-07T15:22:45,036 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:22:45,036 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/C is initiating minor compaction (all files) 2024-11-07T15:22:45,036 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/C in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:45,036 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/6d42596a070c4b3596071655809861cf, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1d856d1166664eb8b78525abf20ea95b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/a34bb7b891534b868e86c1ff7b348a51] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=35.7 K 2024-11-07T15:22:45,037 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d42596a070c4b3596071655809861cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1730992962049 2024-11-07T15:22:45,039 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d856d1166664eb8b78525abf20ea95b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1730992962815 2024-11-07T15:22:45,039 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting a34bb7b891534b868e86c1ff7b348a51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730992963492 2024-11-07T15:22:45,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:45,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:45,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/13733ecd3e2b46ad852fe911ba208ab0 is 50, key is test_row_0/B:col10/1730992964714/Put/seqid=0 2024-11-07T15:22:45,061 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#C#compaction#37 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:45,061 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/90ab5ab273814cf694819521640de4d6 is 50, key is test_row_0/C:col10/1730992964671/Put/seqid=0 2024-11-07T15:22:45,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741873_1049 (size=12151) 2024-11-07T15:22:45,069 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/13733ecd3e2b46ad852fe911ba208ab0 2024-11-07T15:22:45,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741874_1050 (size=12527) 2024-11-07T15:22:45,086 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/90ab5ab273814cf694819521640de4d6 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/90ab5ab273814cf694819521640de4d6 2024-11-07T15:22:45,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/8e3c81a950274dcb87c9df8956f12c20 is 50, key is test_row_0/C:col10/1730992964714/Put/seqid=0 2024-11-07T15:22:45,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741875_1051 (size=12151) 2024-11-07T15:22:45,112 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/C of 86b0da056c7c770742d9ba46450cf601 into 90ab5ab273814cf694819521640de4d6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:45,112 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:45,112 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/C, priority=13, startTime=1730992964921; duration=0sec 2024-11-07T15:22:45,112 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:45,112 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:C 2024-11-07T15:22:45,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993025075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993025078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993025113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993025114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993025114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993025221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993025221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993025231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993025232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993025232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,416 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/1341955a50384398bfd7d7e3333e7376 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/1341955a50384398bfd7d7e3333e7376 2024-11-07T15:22:45,428 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/A of 86b0da056c7c770742d9ba46450cf601 into 1341955a50384398bfd7d7e3333e7376(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:45,428 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:45,428 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/A, priority=13, startTime=1730992964920; duration=0sec 2024-11-07T15:22:45,428 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:45,428 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:A 2024-11-07T15:22:45,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993025431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993025434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993025441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993025441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993025442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,509 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/8e3c81a950274dcb87c9df8956f12c20 2024-11-07T15:22:45,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/7c7e5e5fe7be4ac3afec1cdaddb4fee8 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/7c7e5e5fe7be4ac3afec1cdaddb4fee8 2024-11-07T15:22:45,536 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/7c7e5e5fe7be4ac3afec1cdaddb4fee8, entries=150, sequenceid=191, filesize=11.9 K 2024-11-07T15:22:45,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/13733ecd3e2b46ad852fe911ba208ab0 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/13733ecd3e2b46ad852fe911ba208ab0 2024-11-07T15:22:45,554 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/13733ecd3e2b46ad852fe911ba208ab0, entries=150, sequenceid=191, filesize=11.9 K 2024-11-07T15:22:45,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/8e3c81a950274dcb87c9df8956f12c20 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/8e3c81a950274dcb87c9df8956f12c20 2024-11-07T15:22:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T15:22:45,576 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/8e3c81a950274dcb87c9df8956f12c20, entries=150, sequenceid=191, filesize=11.9 K 2024-11-07T15:22:45,579 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=80.51 KB/82440 for 86b0da056c7c770742d9ba46450cf601 in 588ms, sequenceid=191, compaction requested=false 2024-11-07T15:22:45,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:45,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:45,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-07T15:22:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-07T15:22:45,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-07T15:22:45,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1570 sec 2024-11-07T15:22:45,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.1720 sec 2024-11-07T15:22:45,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:22:45,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T15:22:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:45,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/b2f92d0160274d0680ce4b6c42062865 is 50, key is test_row_0/A:col10/1730992965751/Put/seqid=0 2024-11-07T15:22:45,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741876_1052 (size=14541) 2024-11-07T15:22:45,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/b2f92d0160274d0680ce4b6c42062865 2024-11-07T15:22:45,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/5e14dc22ded548b2b4a0cf35e3e1686c is 50, key is test_row_0/B:col10/1730992965751/Put/seqid=0 2024-11-07T15:22:45,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993025809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993025810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993025811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993025811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993025807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741877_1053 (size=12151) 2024-11-07T15:22:45,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/5e14dc22ded548b2b4a0cf35e3e1686c 2024-11-07T15:22:45,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/1edbba34c34b4c5a93cb40d860f072e7 is 50, key is test_row_0/C:col10/1730992965751/Put/seqid=0 2024-11-07T15:22:45,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741878_1054 (size=12151) 2024-11-07T15:22:45,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993025932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993025932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993025932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993025933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993025933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993026141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993026145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993026145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993026151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993026152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/1edbba34c34b4c5a93cb40d860f072e7 2024-11-07T15:22:46,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/b2f92d0160274d0680ce4b6c42062865 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b2f92d0160274d0680ce4b6c42062865 2024-11-07T15:22:46,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b2f92d0160274d0680ce4b6c42062865, entries=200, sequenceid=210, filesize=14.2 K 2024-11-07T15:22:46,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/5e14dc22ded548b2b4a0cf35e3e1686c as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/5e14dc22ded548b2b4a0cf35e3e1686c 2024-11-07T15:22:46,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/5e14dc22ded548b2b4a0cf35e3e1686c, entries=150, sequenceid=210, filesize=11.9 K 2024-11-07T15:22:46,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/1edbba34c34b4c5a93cb40d860f072e7 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1edbba34c34b4c5a93cb40d860f072e7 2024-11-07T15:22:46,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1edbba34c34b4c5a93cb40d860f072e7, entries=150, sequenceid=210, filesize=11.9 K 2024-11-07T15:22:46,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 86b0da056c7c770742d9ba46450cf601 in 590ms, sequenceid=210, compaction requested=true 2024-11-07T15:22:46,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:46,346 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:22:46,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:22:46,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:46,348 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:22:46,348 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/A is initiating minor compaction (all files) 2024-11-07T15:22:46,349 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/A in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:46,349 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/1341955a50384398bfd7d7e3333e7376, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/7c7e5e5fe7be4ac3afec1cdaddb4fee8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b2f92d0160274d0680ce4b6c42062865] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=38.3 K 2024-11-07T15:22:46,349 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:22:46,350 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1341955a50384398bfd7d7e3333e7376, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730992963492 2024-11-07T15:22:46,350 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c7e5e5fe7be4ac3afec1cdaddb4fee8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1730992964714 2024-11-07T15:22:46,351 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2f92d0160274d0680ce4b6c42062865, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730992965066 2024-11-07T15:22:46,351 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:22:46,351 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/B is initiating minor compaction (all files) 2024-11-07T15:22:46,351 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/B in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:46,351 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8879e2d5ca044cd0a9498bcc5e2db4fc, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/13733ecd3e2b46ad852fe911ba208ab0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/5e14dc22ded548b2b4a0cf35e3e1686c] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=36.0 K 2024-11-07T15:22:46,353 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 8879e2d5ca044cd0a9498bcc5e2db4fc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730992963492 2024-11-07T15:22:46,354 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 13733ecd3e2b46ad852fe911ba208ab0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1730992964714 2024-11-07T15:22:46,355 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e14dc22ded548b2b4a0cf35e3e1686c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730992965066 2024-11-07T15:22:46,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:22:46,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:46,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:22:46,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:46,369 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#A#compaction#42 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:46,369 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/7a261fb2fa4d4195a2cbee8a02600e29 is 50, key is test_row_0/A:col10/1730992965751/Put/seqid=0 2024-11-07T15:22:46,373 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:46,373 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/29a3304aede3432399e6678794819ed4 is 50, key is test_row_0/B:col10/1730992965751/Put/seqid=0 2024-11-07T15:22:46,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741879_1055 (size=12629) 2024-11-07T15:22:46,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741880_1056 (size=12629) 2024-11-07T15:22:46,413 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/7a261fb2fa4d4195a2cbee8a02600e29 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/7a261fb2fa4d4195a2cbee8a02600e29 2024-11-07T15:22:46,422 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/29a3304aede3432399e6678794819ed4 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/29a3304aede3432399e6678794819ed4 2024-11-07T15:22:46,425 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/A of 86b0da056c7c770742d9ba46450cf601 into 7a261fb2fa4d4195a2cbee8a02600e29(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:46,425 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:46,425 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/A, priority=13, startTime=1730992966345; duration=0sec 2024-11-07T15:22:46,425 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:46,425 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:A 2024-11-07T15:22:46,425 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:22:46,427 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:22:46,427 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/C is initiating minor compaction (all files) 2024-11-07T15:22:46,427 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/C in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:46,428 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/90ab5ab273814cf694819521640de4d6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/8e3c81a950274dcb87c9df8956f12c20, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1edbba34c34b4c5a93cb40d860f072e7] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=36.0 K 2024-11-07T15:22:46,429 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90ab5ab273814cf694819521640de4d6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730992963492 2024-11-07T15:22:46,430 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e3c81a950274dcb87c9df8956f12c20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1730992964714 2024-11-07T15:22:46,433 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1edbba34c34b4c5a93cb40d860f072e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730992965066 2024-11-07T15:22:46,437 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/B of 86b0da056c7c770742d9ba46450cf601 into 29a3304aede3432399e6678794819ed4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:46,437 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:46,437 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/B, priority=13, startTime=1730992966349; duration=0sec 2024-11-07T15:22:46,438 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:46,438 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:B 2024-11-07T15:22:46,455 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#C#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:46,456 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/880e5a30f8fd48e6ac652f972c48ce16 is 50, key is test_row_0/C:col10/1730992965751/Put/seqid=0 2024-11-07T15:22:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:46,462 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:22:46,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T15:22:46,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741881_1057 (size=12629) 2024-11-07T15:22:46,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/c087ca492300478c9bc7642a1ba50124 is 50, key is test_row_0/A:col10/1730992965809/Put/seqid=0 2024-11-07T15:22:46,493 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/880e5a30f8fd48e6ac652f972c48ce16 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/880e5a30f8fd48e6ac652f972c48ce16 2024-11-07T15:22:46,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993026486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993026489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993026492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993026492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,508 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/C of 86b0da056c7c770742d9ba46450cf601 into 880e5a30f8fd48e6ac652f972c48ce16(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:46,509 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:46,509 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/C, priority=13, startTime=1730992966362; duration=0sec 2024-11-07T15:22:46,510 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:46,510 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:C 2024-11-07T15:22:46,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993026495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741882_1058 (size=14541) 2024-11-07T15:22:46,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993026601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993026611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993026611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993026612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993026621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993026811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993026821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993026821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993026821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:46,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993026831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:46,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/c087ca492300478c9bc7642a1ba50124 2024-11-07T15:22:46,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/246b2c028e564690b3cc38ca4442bc25 is 50, key is test_row_0/B:col10/1730992965809/Put/seqid=0 2024-11-07T15:22:46,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741883_1059 (size=12151) 2024-11-07T15:22:46,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/246b2c028e564690b3cc38ca4442bc25 2024-11-07T15:22:46,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/748fcc1cb6b54c1580a923c31e991959 is 50, key is test_row_0/C:col10/1730992965809/Put/seqid=0 2024-11-07T15:22:46,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741884_1060 (size=12151) 2024-11-07T15:22:46,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/748fcc1cb6b54c1580a923c31e991959 2024-11-07T15:22:47,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/c087ca492300478c9bc7642a1ba50124 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/c087ca492300478c9bc7642a1ba50124 2024-11-07T15:22:47,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/c087ca492300478c9bc7642a1ba50124, entries=200, sequenceid=234, filesize=14.2 K 2024-11-07T15:22:47,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/246b2c028e564690b3cc38ca4442bc25 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/246b2c028e564690b3cc38ca4442bc25 2024-11-07T15:22:47,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/246b2c028e564690b3cc38ca4442bc25, entries=150, sequenceid=234, filesize=11.9 K 2024-11-07T15:22:47,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/748fcc1cb6b54c1580a923c31e991959 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/748fcc1cb6b54c1580a923c31e991959 2024-11-07T15:22:47,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/748fcc1cb6b54c1580a923c31e991959, entries=150, sequenceid=234, filesize=11.9 K 2024-11-07T15:22:47,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 86b0da056c7c770742d9ba46450cf601 in 577ms, sequenceid=234, compaction requested=false 2024-11-07T15:22:47,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:47,127 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:22:47,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T15:22:47,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/98bb0f3f26ba4ee587d0def0a666894e is 50, key is test_row_0/A:col10/1730992966487/Put/seqid=0 2024-11-07T15:22:47,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741885_1061 (size=12151) 2024-11-07T15:22:47,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/98bb0f3f26ba4ee587d0def0a666894e 2024-11-07T15:22:47,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993027162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993027163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993027163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993027168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993027175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/4d8056adda0941e098463fad4a962efb is 50, key is test_row_0/B:col10/1730992966487/Put/seqid=0 2024-11-07T15:22:47,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741886_1062 (size=12151) 2024-11-07T15:22:47,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/4d8056adda0941e098463fad4a962efb 2024-11-07T15:22:47,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/dfe74fcdef9d4ef9a5f1def7776831d2 is 50, key is test_row_0/C:col10/1730992966487/Put/seqid=0 2024-11-07T15:22:47,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993027281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993027281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993027281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741887_1063 (size=12151) 2024-11-07T15:22:47,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993027292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993027292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993027491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993027491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993027491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993027501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993027501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T15:22:47,591 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:22:47,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:22:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-07T15:22:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T15:22:47,595 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:22:47,596 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:22:47,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:22:47,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/dfe74fcdef9d4ef9a5f1def7776831d2 2024-11-07T15:22:47,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/98bb0f3f26ba4ee587d0def0a666894e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/98bb0f3f26ba4ee587d0def0a666894e 2024-11-07T15:22:47,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/98bb0f3f26ba4ee587d0def0a666894e, entries=150, sequenceid=251, filesize=11.9 K 2024-11-07T15:22:47,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/4d8056adda0941e098463fad4a962efb as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/4d8056adda0941e098463fad4a962efb 2024-11-07T15:22:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T15:22:47,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/4d8056adda0941e098463fad4a962efb, entries=150, sequenceid=251, filesize=11.9 K 2024-11-07T15:22:47,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/dfe74fcdef9d4ef9a5f1def7776831d2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/dfe74fcdef9d4ef9a5f1def7776831d2 2024-11-07T15:22:47,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/dfe74fcdef9d4ef9a5f1def7776831d2, entries=150, sequenceid=251, filesize=11.9 K 2024-11-07T15:22:47,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 86b0da056c7c770742d9ba46450cf601 in 600ms, sequenceid=251, compaction requested=true 2024-11-07T15:22:47,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:47,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:22:47,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:47,728 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:22:47,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:22:47,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:47,728 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:22:47,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:22:47,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:47,729 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:22:47,729 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:22:47,729 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/A is initiating minor compaction (all files) 2024-11-07T15:22:47,729 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/B is initiating minor compaction (all files) 2024-11-07T15:22:47,729 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/B in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:47,729 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/A in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:47,729 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/29a3304aede3432399e6678794819ed4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/246b2c028e564690b3cc38ca4442bc25, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/4d8056adda0941e098463fad4a962efb] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=36.1 K 2024-11-07T15:22:47,729 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/7a261fb2fa4d4195a2cbee8a02600e29, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/c087ca492300478c9bc7642a1ba50124, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/98bb0f3f26ba4ee587d0def0a666894e] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=38.4 K 2024-11-07T15:22:47,730 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a261fb2fa4d4195a2cbee8a02600e29, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730992965066 2024-11-07T15:22:47,730 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 29a3304aede3432399e6678794819ed4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730992965066 2024-11-07T15:22:47,731 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 246b2c028e564690b3cc38ca4442bc25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1730992965808 2024-11-07T15:22:47,732 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting c087ca492300478c9bc7642a1ba50124, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1730992965806 2024-11-07T15:22:47,732 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98bb0f3f26ba4ee587d0def0a666894e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730992966487 2024-11-07T15:22:47,732 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d8056adda0941e098463fad4a962efb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730992966487 2024-11-07T15:22:47,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T15:22:47,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:47,749 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T15:22:47,754 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#B#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:47,755 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/04c31c4948b94cec9ced08c4d0363324 is 50, key is test_row_0/B:col10/1730992966487/Put/seqid=0 2024-11-07T15:22:47,758 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#A#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:47,759 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/2cf849e74d44456aba982dafa53df438 is 50, key is test_row_0/A:col10/1730992966487/Put/seqid=0 2024-11-07T15:22:47,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/191f12fcaf654a9fb7248d29ae44cf79 is 50, key is test_row_0/A:col10/1730992967164/Put/seqid=0 2024-11-07T15:22:47,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741889_1065 (size=12731) 2024-11-07T15:22:47,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741890_1066 (size=12301) 2024-11-07T15:22:47,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741888_1064 (size=12731) 2024-11-07T15:22:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:47,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:47,807 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/2cf849e74d44456aba982dafa53df438 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/2cf849e74d44456aba982dafa53df438 2024-11-07T15:22:47,817 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/A of 86b0da056c7c770742d9ba46450cf601 into 2cf849e74d44456aba982dafa53df438(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:47,817 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:47,817 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/A, priority=13, startTime=1730992967727; duration=0sec 2024-11-07T15:22:47,817 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:47,817 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:A 2024-11-07T15:22:47,817 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:22:47,825 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:22:47,825 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/C is initiating minor compaction (all files) 2024-11-07T15:22:47,825 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/C in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:47,825 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/880e5a30f8fd48e6ac652f972c48ce16, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/748fcc1cb6b54c1580a923c31e991959, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/dfe74fcdef9d4ef9a5f1def7776831d2] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=36.1 K 2024-11-07T15:22:47,826 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 880e5a30f8fd48e6ac652f972c48ce16, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730992965066 2024-11-07T15:22:47,826 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 748fcc1cb6b54c1580a923c31e991959, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1730992965808 2024-11-07T15:22:47,827 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfe74fcdef9d4ef9a5f1def7776831d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730992966487 2024-11-07T15:22:47,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993027841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,849 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#C#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,850 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/53d3317546e84d779f3d3331e509811d is 50, key is test_row_0/C:col10/1730992966487/Put/seqid=0 2024-11-07T15:22:47,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993027846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993027849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993027854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993027854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741891_1067 (size=12731) 2024-11-07T15:22:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T15:22:47,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993027961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993027961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993027971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993027971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993027971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:48,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993028171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:48,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993028171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:48,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:48,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993028180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993028181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993028181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:48,195 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/191f12fcaf654a9fb7248d29ae44cf79 2024-11-07T15:22:48,206 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/04c31c4948b94cec9ced08c4d0363324 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/04c31c4948b94cec9ced08c4d0363324 2024-11-07T15:22:48,213 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/B of 86b0da056c7c770742d9ba46450cf601 into 04c31c4948b94cec9ced08c4d0363324(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:48,213 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:48,213 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/B, priority=13, startTime=1730992967728; duration=0sec 2024-11-07T15:22:48,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/b39316d32462476493c253fa6b70e566 is 50, key is test_row_0/B:col10/1730992967164/Put/seqid=0 2024-11-07T15:22:48,213 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:48,213 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:B 2024-11-07T15:22:48,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741892_1068 (size=12301) 2024-11-07T15:22:48,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T15:22:48,285 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/53d3317546e84d779f3d3331e509811d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/53d3317546e84d779f3d3331e509811d 2024-11-07T15:22:48,291 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/C of 86b0da056c7c770742d9ba46450cf601 into 53d3317546e84d779f3d3331e509811d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:48,291 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:48,291 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/C, priority=13, startTime=1730992967728; duration=0sec 2024-11-07T15:22:48,291 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:48,291 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:C 2024-11-07T15:22:48,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993028481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:48,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993028481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:48,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:48,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993028491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:48,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:48,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993028491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:48,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:48,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993028491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:48,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:48,619 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/b39316d32462476493c253fa6b70e566 2024-11-07T15:22:48,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/9e8aae2433db4f1ea37c0b548c2c63c7 is 50, key is test_row_0/C:col10/1730992967164/Put/seqid=0 2024-11-07T15:22:48,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741893_1069 (size=12301) 2024-11-07T15:22:48,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T15:22:48,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993028990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:48,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:48,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993028990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:48,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:48,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:49,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993029001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:49,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:49,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:49,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:49,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993029001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:49,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:49,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:49,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:49,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:49,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993029001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:49,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:49,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:49,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:49,039 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/9e8aae2433db4f1ea37c0b548c2c63c7 2024-11-07T15:22:49,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/191f12fcaf654a9fb7248d29ae44cf79 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/191f12fcaf654a9fb7248d29ae44cf79 2024-11-07T15:22:49,057 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/191f12fcaf654a9fb7248d29ae44cf79, entries=150, sequenceid=271, filesize=12.0 K 2024-11-07T15:22:49,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/b39316d32462476493c253fa6b70e566 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b39316d32462476493c253fa6b70e566 2024-11-07T15:22:49,067 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b39316d32462476493c253fa6b70e566, entries=150, sequenceid=271, filesize=12.0 K 2024-11-07T15:22:49,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/9e8aae2433db4f1ea37c0b548c2c63c7 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/9e8aae2433db4f1ea37c0b548c2c63c7 2024-11-07T15:22:49,077 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/9e8aae2433db4f1ea37c0b548c2c63c7, entries=150, sequenceid=271, filesize=12.0 K 2024-11-07T15:22:49,078 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=100.63 KB/103050 for 86b0da056c7c770742d9ba46450cf601 in 1329ms, sequenceid=271, compaction requested=false 2024-11-07T15:22:49,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:49,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:49,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-07T15:22:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-07T15:22:49,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-07T15:22:49,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4900 sec 2024-11-07T15:22:49,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.5000 sec 2024-11-07T15:22:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T15:22:49,742 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:22:49,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:22:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-07T15:22:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T15:22:49,748 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:22:49,749 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:22:49,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:22:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T15:22:49,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T15:22:49,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:49,905 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T15:22:49,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/0755b5a650e24f77afd9de75a86d029e is 50, key is test_row_0/A:col10/1730992967845/Put/seqid=0 2024-11-07T15:22:49,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741894_1070 (size=12301) 2024-11-07T15:22:50,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:50,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993030036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993030040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993030040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993030042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993030043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T15:22:50,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993030151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993030151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993030152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993030152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993030152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,347 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/0755b5a650e24f77afd9de75a86d029e 2024-11-07T15:22:50,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/8b557a1316d34be6a3fe13ada5f55ca7 is 50, key is test_row_0/B:col10/1730992967845/Put/seqid=0 2024-11-07T15:22:50,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993030361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T15:22:50,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993030362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993030362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993030362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993030362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741895_1071 (size=12301) 2024-11-07T15:22:50,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993030681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993030681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993030681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993030681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993030701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:50,817 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/8b557a1316d34be6a3fe13ada5f55ca7 2024-11-07T15:22:50,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/d518f060439b4548abf228b0b01ba481 is 50, key is test_row_0/C:col10/1730992967845/Put/seqid=0 2024-11-07T15:22:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T15:22:50,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741896_1072 (size=12301) 2024-11-07T15:22:51,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993031191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:51,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993031191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:51,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993031191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:51,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993031191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:51,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993031211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:51,284 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/d518f060439b4548abf228b0b01ba481 2024-11-07T15:22:51,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/0755b5a650e24f77afd9de75a86d029e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/0755b5a650e24f77afd9de75a86d029e 2024-11-07T15:22:51,321 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/0755b5a650e24f77afd9de75a86d029e, entries=150, sequenceid=292, filesize=12.0 K 2024-11-07T15:22:51,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/8b557a1316d34be6a3fe13ada5f55ca7 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8b557a1316d34be6a3fe13ada5f55ca7 2024-11-07T15:22:51,332 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8b557a1316d34be6a3fe13ada5f55ca7, entries=150, sequenceid=292, filesize=12.0 K 2024-11-07T15:22:51,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/d518f060439b4548abf228b0b01ba481 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/d518f060439b4548abf228b0b01ba481 2024-11-07T15:22:51,350 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/d518f060439b4548abf228b0b01ba481, entries=150, sequenceid=292, filesize=12.0 K 2024-11-07T15:22:51,351 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 86b0da056c7c770742d9ba46450cf601 in 1447ms, sequenceid=292, compaction requested=true 2024-11-07T15:22:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-07T15:22:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-07T15:22:51,357 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-07T15:22:51,357 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6050 sec 2024-11-07T15:22:51,361 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.6170 sec 2024-11-07T15:22:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T15:22:51,891 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:22:51,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:22:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-07T15:22:51,895 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:22:51,896 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:22:51,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:22:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T15:22:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T15:22:52,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T15:22:52,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:52,050 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T15:22:52,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/371be59ec416441f86117ef2952a0e76 is 50, key is test_row_0/A:col10/1730992970039/Put/seqid=0 2024-11-07T15:22:52,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741897_1073 (size=12301) 2024-11-07T15:22:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:52,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T15:22:52,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993032253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993032254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993032254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993032256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993032262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993032371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993032372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993032372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993032372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993032372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,471 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/371be59ec416441f86117ef2952a0e76 2024-11-07T15:22:52,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/94c36ce7ddc54505b604cbbef2484ecc is 50, key is test_row_0/B:col10/1730992970039/Put/seqid=0 2024-11-07T15:22:52,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741898_1074 (size=12301) 2024-11-07T15:22:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T15:22:52,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993032581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993032581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993032581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993032591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993032591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993032892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993032892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993032893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993032901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993032904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:52,917 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/94c36ce7ddc54505b604cbbef2484ecc 2024-11-07T15:22:52,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/4198544e0ed140c682baf33f950d9c62 is 50, key is test_row_0/C:col10/1730992970039/Put/seqid=0 2024-11-07T15:22:52,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741899_1075 (size=12301) 2024-11-07T15:22:52,956 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/4198544e0ed140c682baf33f950d9c62 2024-11-07T15:22:52,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/371be59ec416441f86117ef2952a0e76 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/371be59ec416441f86117ef2952a0e76 2024-11-07T15:22:52,985 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/371be59ec416441f86117ef2952a0e76, entries=150, sequenceid=310, filesize=12.0 K 2024-11-07T15:22:52,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/94c36ce7ddc54505b604cbbef2484ecc as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/94c36ce7ddc54505b604cbbef2484ecc 2024-11-07T15:22:53,002 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/94c36ce7ddc54505b604cbbef2484ecc, entries=150, sequenceid=310, filesize=12.0 K 2024-11-07T15:22:53,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/4198544e0ed140c682baf33f950d9c62 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/4198544e0ed140c682baf33f950d9c62 2024-11-07T15:22:53,018 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/4198544e0ed140c682baf33f950d9c62, entries=150, sequenceid=310, filesize=12.0 K 2024-11-07T15:22:53,027 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 86b0da056c7c770742d9ba46450cf601 in 977ms, sequenceid=310, compaction requested=true 2024-11-07T15:22:53,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:53,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:53,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-07T15:22:53,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-07T15:22:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T15:22:53,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-07T15:22:53,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1340 sec 2024-11-07T15:22:53,046 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.1450 sec 2024-11-07T15:22:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:53,405 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:22:53,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T15:22:53,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/b148e4ce2eeb452fa77c51a66efc890f is 50, key is test_row_0/A:col10/1730992972230/Put/seqid=0 2024-11-07T15:22:53,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741900_1076 (size=17181) 2024-11-07T15:22:53,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:53,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:53,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993033426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:53,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993033427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:53,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:53,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993033468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:53,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993033470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:53,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:53,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993033471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:53,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993033570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:53,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:53,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993033581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:53,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:53,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993033581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:53,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:53,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:53,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993033582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:53,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:53,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:53,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993033582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:53,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:53,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993033781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:53,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:53,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993033791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993033791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:53,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993033791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:53,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993033791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:53,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/b148e4ce2eeb452fa77c51a66efc890f 2024-11-07T15:22:53,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/6863b5d230db4759b0495c832a338b05 is 50, key is test_row_0/B:col10/1730992972230/Put/seqid=0 2024-11-07T15:22:53,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741901_1077 (size=12301) 2024-11-07T15:22:53,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/6863b5d230db4759b0495c832a338b05 2024-11-07T15:22:53,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/6235ceffaba646449e759a4199f0110d is 50, key is test_row_0/C:col10/1730992972230/Put/seqid=0 2024-11-07T15:22:53,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741902_1078 (size=12301) 2024-11-07T15:22:53,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/6235ceffaba646449e759a4199f0110d 2024-11-07T15:22:53,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/b148e4ce2eeb452fa77c51a66efc890f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b148e4ce2eeb452fa77c51a66efc890f 2024-11-07T15:22:53,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b148e4ce2eeb452fa77c51a66efc890f, entries=250, sequenceid=330, filesize=16.8 K 2024-11-07T15:22:53,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/6863b5d230db4759b0495c832a338b05 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/6863b5d230db4759b0495c832a338b05 2024-11-07T15:22:53,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/6863b5d230db4759b0495c832a338b05, entries=150, sequenceid=330, filesize=12.0 K 2024-11-07T15:22:53,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/6235ceffaba646449e759a4199f0110d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/6235ceffaba646449e759a4199f0110d 2024-11-07T15:22:53,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/6235ceffaba646449e759a4199f0110d, entries=150, sequenceid=330, filesize=12.0 K 2024-11-07T15:22:53,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 86b0da056c7c770742d9ba46450cf601 in 582ms, sequenceid=330, compaction requested=true 2024-11-07T15:22:53,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:53,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:22:53,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:53,987 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T15:22:53,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:22:53,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:53,987 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T15:22:53,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:22:53,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:53,989 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 66815 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T15:22:53,990 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/A is initiating minor compaction (all files) 2024-11-07T15:22:53,990 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/A in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:53,990 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/2cf849e74d44456aba982dafa53df438, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/191f12fcaf654a9fb7248d29ae44cf79, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/0755b5a650e24f77afd9de75a86d029e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/371be59ec416441f86117ef2952a0e76, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b148e4ce2eeb452fa77c51a66efc890f] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=65.2 K 2024-11-07T15:22:53,990 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61935 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T15:22:53,990 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/B is initiating minor compaction (all files) 2024-11-07T15:22:53,991 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/B in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:53,991 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/04c31c4948b94cec9ced08c4d0363324, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b39316d32462476493c253fa6b70e566, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8b557a1316d34be6a3fe13ada5f55ca7, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/94c36ce7ddc54505b604cbbef2484ecc, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/6863b5d230db4759b0495c832a338b05] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=60.5 K 2024-11-07T15:22:53,991 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cf849e74d44456aba982dafa53df438, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730992966487 2024-11-07T15:22:53,991 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 04c31c4948b94cec9ced08c4d0363324, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730992966487 2024-11-07T15:22:53,992 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 191f12fcaf654a9fb7248d29ae44cf79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1730992967156 2024-11-07T15:22:53,992 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting b39316d32462476493c253fa6b70e566, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1730992967156 2024-11-07T15:22:53,992 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0755b5a650e24f77afd9de75a86d029e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1730992967845 2024-11-07T15:22:53,992 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b557a1316d34be6a3fe13ada5f55ca7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1730992967845 2024-11-07T15:22:53,993 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 371be59ec416441f86117ef2952a0e76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1730992970015 2024-11-07T15:22:53,993 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 94c36ce7ddc54505b604cbbef2484ecc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1730992970015 2024-11-07T15:22:53,993 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting b148e4ce2eeb452fa77c51a66efc890f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1730992972227 2024-11-07T15:22:53,993 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 6863b5d230db4759b0495c832a338b05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1730992972230 2024-11-07T15:22:54,031 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#B#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:54,032 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/faaf9033735f41d198fc1b5300e2c4a1 is 50, key is test_row_0/B:col10/1730992972230/Put/seqid=0 2024-11-07T15:22:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T15:22:54,042 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:22:54,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:22:54,043 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#A#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-07T15:22:54,045 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/e9de5cf9dd3c4df598bfb5b10cd22574 is 50, key is test_row_0/A:col10/1730992972230/Put/seqid=0 2024-11-07T15:22:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T15:22:54,046 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:22:54,047 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:22:54,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:22:54,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741903_1079 (size=13051) 2024-11-07T15:22:54,103 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/faaf9033735f41d198fc1b5300e2c4a1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/faaf9033735f41d198fc1b5300e2c4a1 2024-11-07T15:22:54,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741904_1080 (size=13051) 2024-11-07T15:22:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:22:54,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-07T15:22:54,116 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/e9de5cf9dd3c4df598bfb5b10cd22574 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/e9de5cf9dd3c4df598bfb5b10cd22574 2024-11-07T15:22:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:54,118 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/B of 86b0da056c7c770742d9ba46450cf601 into faaf9033735f41d198fc1b5300e2c4a1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:54,118 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:54,118 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/B, priority=11, startTime=1730992973987; duration=0sec 2024-11-07T15:22:54,118 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:54,118 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:B 2024-11-07T15:22:54,119 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T15:22:54,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/db8797a21b7140ca9e656c377ec3b959 is 50, key is test_row_0/A:col10/1730992973429/Put/seqid=0 2024-11-07T15:22:54,123 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61935 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T15:22:54,123 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/C is initiating minor compaction (all files) 2024-11-07T15:22:54,123 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/C in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:54,124 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/53d3317546e84d779f3d3331e509811d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/9e8aae2433db4f1ea37c0b548c2c63c7, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/d518f060439b4548abf228b0b01ba481, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/4198544e0ed140c682baf33f950d9c62, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/6235ceffaba646449e759a4199f0110d] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=60.5 K 2024-11-07T15:22:54,125 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 53d3317546e84d779f3d3331e509811d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730992966487 2024-11-07T15:22:54,133 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e8aae2433db4f1ea37c0b548c2c63c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1730992967156 2024-11-07T15:22:54,134 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting d518f060439b4548abf228b0b01ba481, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1730992967845 2024-11-07T15:22:54,136 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 4198544e0ed140c682baf33f950d9c62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1730992970015 2024-11-07T15:22:54,136 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 6235ceffaba646449e759a4199f0110d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1730992972230 2024-11-07T15:22:54,137 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/A of 86b0da056c7c770742d9ba46450cf601 into e9de5cf9dd3c4df598bfb5b10cd22574(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:54,137 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:54,137 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/A, priority=11, startTime=1730992973987; duration=0sec 2024-11-07T15:22:54,137 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:54,137 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:A 2024-11-07T15:22:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T15:22:54,172 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:54,173 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/54f42d03e00b44b69a9dc83d260968a3 is 50, key is test_row_0/C:col10/1730992972230/Put/seqid=0 2024-11-07T15:22:54,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993034160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741905_1081 (size=12301) 2024-11-07T15:22:54,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993034170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993034176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993034179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993034177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T15:22:54,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:54,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:54,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:54,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741906_1082 (size=13051) 2024-11-07T15:22:54,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993034282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993034293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993034302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993034302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993034303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T15:22:54,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:54,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:54,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:54,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T15:22:54,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993034500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T15:22:54,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:54,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:54,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:54,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993034511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993034511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993034511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993034521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/db8797a21b7140ca9e656c377ec3b959 2024-11-07T15:22:54,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/d4e7d723bb9d46dc88cfde240afbcf49 is 50, key is test_row_0/B:col10/1730992973429/Put/seqid=0 2024-11-07T15:22:54,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741907_1083 (size=12301) 2024-11-07T15:22:54,630 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/54f42d03e00b44b69a9dc83d260968a3 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/54f42d03e00b44b69a9dc83d260968a3 2024-11-07T15:22:54,640 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/C of 86b0da056c7c770742d9ba46450cf601 into 54f42d03e00b44b69a9dc83d260968a3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:54,640 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:54,640 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/C, priority=11, startTime=1730992973987; duration=0sec 2024-11-07T15:22:54,640 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:54,640 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:C 2024-11-07T15:22:54,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T15:22:54,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:54,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:54,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:54,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T15:22:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993034820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993034821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993034821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993034821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T15:22:54,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:54,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:54,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:54,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993034830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:54,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T15:22:54,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:54,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:54,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:54,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:55,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/d4e7d723bb9d46dc88cfde240afbcf49 2024-11-07T15:22:55,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/1a4dd5963b6b4ab7b701622b0e523f71 is 50, key is test_row_0/C:col10/1730992973429/Put/seqid=0 2024-11-07T15:22:55,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741908_1084 (size=12301) 2024-11-07T15:22:55,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/1a4dd5963b6b4ab7b701622b0e523f71 2024-11-07T15:22:55,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/db8797a21b7140ca9e656c377ec3b959 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/db8797a21b7140ca9e656c377ec3b959 2024-11-07T15:22:55,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/db8797a21b7140ca9e656c377ec3b959, entries=150, sequenceid=347, filesize=12.0 K 2024-11-07T15:22:55,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/d4e7d723bb9d46dc88cfde240afbcf49 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/d4e7d723bb9d46dc88cfde240afbcf49 2024-11-07T15:22:55,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/d4e7d723bb9d46dc88cfde240afbcf49, entries=150, sequenceid=347, filesize=12.0 K 2024-11-07T15:22:55,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/1a4dd5963b6b4ab7b701622b0e523f71 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1a4dd5963b6b4ab7b701622b0e523f71 2024-11-07T15:22:55,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1a4dd5963b6b4ab7b701622b0e523f71, entries=150, sequenceid=347, filesize=12.0 K 2024-11-07T15:22:55,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 86b0da056c7c770742d9ba46450cf601 in 1036ms, sequenceid=347, compaction requested=false 2024-11-07T15:22:55,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:55,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T15:22:55,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:55,146 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T15:22:55,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/b55f98a912964fd0bb2cfbff7abb68b8 is 50, key is test_row_0/A:col10/1730992974174/Put/seqid=0 2024-11-07T15:22:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T15:22:55,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741909_1085 (size=12301) 2024-11-07T15:22:55,195 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/b55f98a912964fd0bb2cfbff7abb68b8 2024-11-07T15:22:55,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/5349722ed979470b9963d7201411ed26 is 50, key is test_row_0/B:col10/1730992974174/Put/seqid=0 2024-11-07T15:22:55,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741910_1086 (size=12301) 2024-11-07T15:22:55,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:55,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:55,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993035359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993035361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:55,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993035368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:55,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993035369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993035369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:55,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993035481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:55,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:55,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993035482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:55,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:55,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:55,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:55,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993035482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993035482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:55,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993035482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:55,670 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/5349722ed979470b9963d7201411ed26 2024-11-07T15:22:55,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/b19b472fec984c3ea6af58a8efc65d89 is 50, key is test_row_0/C:col10/1730992974174/Put/seqid=0 2024-11-07T15:22:55,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:55,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993035691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993035691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:55,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993035691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993035691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:55,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993035701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:55,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741911_1087 (size=12301) 2024-11-07T15:22:55,721 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/b19b472fec984c3ea6af58a8efc65d89 2024-11-07T15:22:55,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/b55f98a912964fd0bb2cfbff7abb68b8 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b55f98a912964fd0bb2cfbff7abb68b8 2024-11-07T15:22:55,749 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b55f98a912964fd0bb2cfbff7abb68b8, entries=150, sequenceid=370, filesize=12.0 K 2024-11-07T15:22:55,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/5349722ed979470b9963d7201411ed26 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/5349722ed979470b9963d7201411ed26 2024-11-07T15:22:55,759 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/5349722ed979470b9963d7201411ed26, entries=150, sequenceid=370, filesize=12.0 K 2024-11-07T15:22:55,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/b19b472fec984c3ea6af58a8efc65d89 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b19b472fec984c3ea6af58a8efc65d89 2024-11-07T15:22:55,773 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b19b472fec984c3ea6af58a8efc65d89, entries=150, sequenceid=370, filesize=12.0 K 2024-11-07T15:22:55,776 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 86b0da056c7c770742d9ba46450cf601 in 630ms, sequenceid=370, compaction requested=true 2024-11-07T15:22:55,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:55,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:55,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-07T15:22:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-07T15:22:55,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-07T15:22:55,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7320 sec 2024-11-07T15:22:55,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.7400 sec 2024-11-07T15:22:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:56,013 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:22:56,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T15:22:56,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/404f9a3802264884abe45edd529399fd is 50, key is test_row_0/A:col10/1730992976011/Put/seqid=0 2024-11-07T15:22:56,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993036043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993036045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993036049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993036057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993036063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741912_1088 (size=14741) 2024-11-07T15:22:56,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/404f9a3802264884abe45edd529399fd 2024-11-07T15:22:56,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/f548367c027842b6adcded547bddf756 is 50, key is test_row_0/B:col10/1730992976011/Put/seqid=0 2024-11-07T15:22:56,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741913_1089 (size=12301) 2024-11-07T15:22:56,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/f548367c027842b6adcded547bddf756 2024-11-07T15:22:56,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/c6313bc25fb24dc78e67fe61d256d8a6 is 50, key is test_row_0/C:col10/1730992976011/Put/seqid=0 2024-11-07T15:22:56,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741914_1090 (size=12301) 2024-11-07T15:22:56,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/c6313bc25fb24dc78e67fe61d256d8a6 2024-11-07T15:22:56,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/404f9a3802264884abe45edd529399fd as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/404f9a3802264884abe45edd529399fd 2024-11-07T15:22:56,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/404f9a3802264884abe45edd529399fd, entries=200, sequenceid=388, filesize=14.4 K 2024-11-07T15:22:56,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/f548367c027842b6adcded547bddf756 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/f548367c027842b6adcded547bddf756 2024-11-07T15:22:56,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993036171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993036171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993036171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993036176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/f548367c027842b6adcded547bddf756, entries=150, sequenceid=388, filesize=12.0 K 2024-11-07T15:22:56,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/c6313bc25fb24dc78e67fe61d256d8a6 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/c6313bc25fb24dc78e67fe61d256d8a6 2024-11-07T15:22:56,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993036181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T15:22:56,202 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:22:56,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:22:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-07T15:22:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T15:22:56,207 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:22:56,209 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:22:56,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:22:56,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/c6313bc25fb24dc78e67fe61d256d8a6, entries=150, sequenceid=388, filesize=12.0 K 2024-11-07T15:22:56,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 86b0da056c7c770742d9ba46450cf601 in 202ms, sequenceid=388, compaction requested=true 2024-11-07T15:22:56,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:56,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:22:56,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:56,216 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:22:56,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:22:56,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:56,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:22:56,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T15:22:56,216 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:22:56,219 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:22:56,219 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/A is initiating minor compaction (all files) 2024-11-07T15:22:56,219 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/A in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:56,219 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/e9de5cf9dd3c4df598bfb5b10cd22574, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/db8797a21b7140ca9e656c377ec3b959, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b55f98a912964fd0bb2cfbff7abb68b8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/404f9a3802264884abe45edd529399fd] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=51.2 K 2024-11-07T15:22:56,220 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:22:56,220 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/B is initiating minor compaction (all files) 2024-11-07T15:22:56,220 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/B in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:56,220 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/faaf9033735f41d198fc1b5300e2c4a1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/d4e7d723bb9d46dc88cfde240afbcf49, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/5349722ed979470b9963d7201411ed26, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/f548367c027842b6adcded547bddf756] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=48.8 K 2024-11-07T15:22:56,220 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting e9de5cf9dd3c4df598bfb5b10cd22574, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1730992972230 2024-11-07T15:22:56,221 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting faaf9033735f41d198fc1b5300e2c4a1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1730992972230 2024-11-07T15:22:56,221 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting db8797a21b7140ca9e656c377ec3b959, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1730992973429 2024-11-07T15:22:56,221 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4e7d723bb9d46dc88cfde240afbcf49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1730992973429 2024-11-07T15:22:56,222 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5349722ed979470b9963d7201411ed26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1730992974167 2024-11-07T15:22:56,222 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting b55f98a912964fd0bb2cfbff7abb68b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1730992974167 2024-11-07T15:22:56,222 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting f548367c027842b6adcded547bddf756, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1730992975367 2024-11-07T15:22:56,222 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 404f9a3802264884abe45edd529399fd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1730992975365 2024-11-07T15:22:56,255 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:56,255 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/49f26e848ab44ce0b097b04852d324cc is 50, key is test_row_0/A:col10/1730992976011/Put/seqid=0 2024-11-07T15:22:56,258 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#B#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:56,263 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/356ca9b168ec42d7b80af7c6745c1483 is 50, key is test_row_0/B:col10/1730992976011/Put/seqid=0 2024-11-07T15:22:56,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741915_1091 (size=13187) 2024-11-07T15:22:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T15:22:56,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741916_1092 (size=13187) 2024-11-07T15:22:56,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T15:22:56,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:56,368 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T15:22:56,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:56,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:56,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/466eba214d4d41eb8ac3c9b3452920d2 is 50, key is test_row_0/A:col10/1730992976042/Put/seqid=0 2024-11-07T15:22:56,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993036413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993036414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993036414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993036416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741917_1093 (size=12301) 2024-11-07T15:22:56,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993036421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T15:22:56,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993036541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993036541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993036542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993036542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993036550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,712 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/356ca9b168ec42d7b80af7c6745c1483 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/356ca9b168ec42d7b80af7c6745c1483 2024-11-07T15:22:56,731 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/B of 86b0da056c7c770742d9ba46450cf601 into 356ca9b168ec42d7b80af7c6745c1483(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:56,731 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:56,731 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/B, priority=12, startTime=1730992976216; duration=0sec 2024-11-07T15:22:56,732 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:56,732 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:B 2024-11-07T15:22:56,732 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:22:56,734 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:22:56,734 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/C is initiating minor compaction (all files) 2024-11-07T15:22:56,734 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/C in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:56,734 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/54f42d03e00b44b69a9dc83d260968a3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1a4dd5963b6b4ab7b701622b0e523f71, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b19b472fec984c3ea6af58a8efc65d89, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/c6313bc25fb24dc78e67fe61d256d8a6] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=48.8 K 2024-11-07T15:22:56,737 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54f42d03e00b44b69a9dc83d260968a3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1730992972230 2024-11-07T15:22:56,738 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a4dd5963b6b4ab7b701622b0e523f71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1730992973429 2024-11-07T15:22:56,739 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting b19b472fec984c3ea6af58a8efc65d89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1730992974167 2024-11-07T15:22:56,740 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/49f26e848ab44ce0b097b04852d324cc as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/49f26e848ab44ce0b097b04852d324cc 2024-11-07T15:22:56,740 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6313bc25fb24dc78e67fe61d256d8a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1730992975367 2024-11-07T15:22:56,748 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/A of 86b0da056c7c770742d9ba46450cf601 into 49f26e848ab44ce0b097b04852d324cc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:56,748 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:56,748 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/A, priority=12, startTime=1730992976216; duration=0sec 2024-11-07T15:22:56,749 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:56,749 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:A 2024-11-07T15:22:56,757 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#C#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:56,758 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/7e6ce7d4739246cab3591951020a8af2 is 50, key is test_row_0/C:col10/1730992976011/Put/seqid=0 2024-11-07T15:22:56,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993036751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993036751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993036751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993036760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993036761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:56,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:56,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:56,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741918_1094 (size=13187) 2024-11-07T15:22:56,805 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/7e6ce7d4739246cab3591951020a8af2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/7e6ce7d4739246cab3591951020a8af2 2024-11-07T15:22:56,815 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/C of 86b0da056c7c770742d9ba46450cf601 into 7e6ce7d4739246cab3591951020a8af2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:56,815 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:56,815 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/C, priority=12, startTime=1730992976216; duration=0sec 2024-11-07T15:22:56,815 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:56,815 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:C 2024-11-07T15:22:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T15:22:56,834 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/466eba214d4d41eb8ac3c9b3452920d2 2024-11-07T15:22:56,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/b186ad9e49b94fb49eb291765ec819d6 is 50, key is test_row_0/B:col10/1730992976042/Put/seqid=0 2024-11-07T15:22:56,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741919_1095 (size=12301) 2024-11-07T15:22:56,908 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/b186ad9e49b94fb49eb291765ec819d6 2024-11-07T15:22:56,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/d6d93a2cc3f64495b5134dfdf184b549 is 50, key is test_row_0/C:col10/1730992976042/Put/seqid=0 2024-11-07T15:22:56,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741920_1096 (size=12301) 2024-11-07T15:22:56,975 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/d6d93a2cc3f64495b5134dfdf184b549 2024-11-07T15:22:56,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/466eba214d4d41eb8ac3c9b3452920d2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/466eba214d4d41eb8ac3c9b3452920d2 2024-11-07T15:22:57,006 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/466eba214d4d41eb8ac3c9b3452920d2, entries=150, sequenceid=407, filesize=12.0 K 2024-11-07T15:22:57,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/b186ad9e49b94fb49eb291765ec819d6 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b186ad9e49b94fb49eb291765ec819d6 2024-11-07T15:22:57,022 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b186ad9e49b94fb49eb291765ec819d6, entries=150, sequenceid=407, filesize=12.0 K 2024-11-07T15:22:57,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/d6d93a2cc3f64495b5134dfdf184b549 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/d6d93a2cc3f64495b5134dfdf184b549 2024-11-07T15:22:57,032 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/d6d93a2cc3f64495b5134dfdf184b549, entries=150, sequenceid=407, filesize=12.0 K 2024-11-07T15:22:57,034 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 86b0da056c7c770742d9ba46450cf601 in 665ms, sequenceid=407, compaction requested=false 2024-11-07T15:22:57,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:57,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:57,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-07T15:22:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-07T15:22:57,041 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-07T15:22:57,041 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 827 msec 2024-11-07T15:22:57,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 839 msec 2024-11-07T15:22:57,077 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:22:57,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T15:22:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:57,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/a93a74f766d14213b78da1f4428346cd is 50, key is test_row_0/A:col10/1730992977075/Put/seqid=0 2024-11-07T15:22:57,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741921_1097 (size=12301) 2024-11-07T15:22:57,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/a93a74f766d14213b78da1f4428346cd 2024-11-07T15:22:57,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993037121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993037127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993037132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993037132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993037134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/4536402980c1408983d6e0a7be35b1c4 is 50, key is test_row_0/B:col10/1730992977075/Put/seqid=0 2024-11-07T15:22:57,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741922_1098 (size=12301) 2024-11-07T15:22:57,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993037242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993037251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993037251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993037252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993037252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T15:22:57,341 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:22:57,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:22:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-07T15:22:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T15:22:57,343 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:22:57,344 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:22:57,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:22:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T15:22:57,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993037451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993037460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993037460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993037471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993037471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T15:22:57,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:57,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:57,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:57,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:57,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:57,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/4536402980c1408983d6e0a7be35b1c4 2024-11-07T15:22:57,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/bd8913271ade420db14f9e3fb7740cff is 50, key is test_row_0/C:col10/1730992977075/Put/seqid=0 2024-11-07T15:22:57,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741923_1099 (size=12301) 2024-11-07T15:22:57,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T15:22:57,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:57,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:57,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:57,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:57,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T15:22:57,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993037761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993037772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993037772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993037781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:57,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993037781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:57,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:57,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:57,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T15:22:57,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:57,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:57,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:57,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:57,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:57,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T15:22:57,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:57,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:57,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:57,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:57,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:22:57,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T15:22:58,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/bd8913271ade420db14f9e3fb7740cff 2024-11-07T15:22:58,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/a93a74f766d14213b78da1f4428346cd as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/a93a74f766d14213b78da1f4428346cd 2024-11-07T15:22:58,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/a93a74f766d14213b78da1f4428346cd, entries=150, sequenceid=430, filesize=12.0 K 2024-11-07T15:22:58,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/4536402980c1408983d6e0a7be35b1c4 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/4536402980c1408983d6e0a7be35b1c4 2024-11-07T15:22:58,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/4536402980c1408983d6e0a7be35b1c4, entries=150, sequenceid=430, filesize=12.0 K 2024-11-07T15:22:58,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/bd8913271ade420db14f9e3fb7740cff as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/bd8913271ade420db14f9e3fb7740cff 2024-11-07T15:22:58,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/bd8913271ade420db14f9e3fb7740cff, entries=150, sequenceid=430, filesize=12.0 K 2024-11-07T15:22:58,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 86b0da056c7c770742d9ba46450cf601 in 1000ms, sequenceid=430, compaction requested=true 2024-11-07T15:22:58,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:58,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:22:58,078 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:22:58,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:58,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:22:58,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:58,078 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:22:58,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:22:58,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:58,079 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:22:58,079 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:22:58,080 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/B is initiating minor compaction (all files) 2024-11-07T15:22:58,080 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/A is initiating minor compaction (all files) 2024-11-07T15:22:58,080 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/A in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:58,080 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/B in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:58,080 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/356ca9b168ec42d7b80af7c6745c1483, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b186ad9e49b94fb49eb291765ec819d6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/4536402980c1408983d6e0a7be35b1c4] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=36.9 K 2024-11-07T15:22:58,080 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/49f26e848ab44ce0b097b04852d324cc, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/466eba214d4d41eb8ac3c9b3452920d2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/a93a74f766d14213b78da1f4428346cd] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=36.9 K 2024-11-07T15:22:58,081 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 356ca9b168ec42d7b80af7c6745c1483, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1730992975367 2024-11-07T15:22:58,081 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49f26e848ab44ce0b097b04852d324cc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1730992975367 2024-11-07T15:22:58,081 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 466eba214d4d41eb8ac3c9b3452920d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1730992976042 2024-11-07T15:22:58,081 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting b186ad9e49b94fb49eb291765ec819d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1730992976042 2024-11-07T15:22:58,082 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 4536402980c1408983d6e0a7be35b1c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1730992976402 2024-11-07T15:22:58,082 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting a93a74f766d14213b78da1f4428346cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1730992976402 2024-11-07T15:22:58,100 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:58,100 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/a0effd952f3f4258a4618a8f3d32a716 is 50, key is test_row_0/A:col10/1730992977075/Put/seqid=0 2024-11-07T15:22:58,108 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#B#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:58,109 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/d403dbf73f8d4b7284ee5e8dd8bb4968 is 50, key is test_row_0/B:col10/1730992977075/Put/seqid=0 2024-11-07T15:22:58,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T15:22:58,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:58,123 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T15:22:58,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/4f7d907b8e5f4b7aac3c34a39b279672 is 50, key is test_row_0/A:col10/1730992977121/Put/seqid=0 2024-11-07T15:22:58,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741924_1100 (size=13289) 2024-11-07T15:22:58,170 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/a0effd952f3f4258a4618a8f3d32a716 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/a0effd952f3f4258a4618a8f3d32a716 2024-11-07T15:22:58,186 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/A of 86b0da056c7c770742d9ba46450cf601 into a0effd952f3f4258a4618a8f3d32a716(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:58,186 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:58,186 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/A, priority=13, startTime=1730992978078; duration=0sec 2024-11-07T15:22:58,186 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:22:58,186 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:A 2024-11-07T15:22:58,186 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:22:58,189 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:22:58,190 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/C is initiating minor compaction (all files) 2024-11-07T15:22:58,190 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/C in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:58,190 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/7e6ce7d4739246cab3591951020a8af2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/d6d93a2cc3f64495b5134dfdf184b549, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/bd8913271ade420db14f9e3fb7740cff] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=36.9 K 2024-11-07T15:22:58,191 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e6ce7d4739246cab3591951020a8af2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1730992975367 2024-11-07T15:22:58,191 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6d93a2cc3f64495b5134dfdf184b549, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1730992976042 2024-11-07T15:22:58,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741925_1101 (size=13289) 2024-11-07T15:22:58,192 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd8913271ade420db14f9e3fb7740cff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1730992976402 2024-11-07T15:22:58,200 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/d403dbf73f8d4b7284ee5e8dd8bb4968 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/d403dbf73f8d4b7284ee5e8dd8bb4968 2024-11-07T15:22:58,214 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/B of 86b0da056c7c770742d9ba46450cf601 into d403dbf73f8d4b7284ee5e8dd8bb4968(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:58,214 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:58,214 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/B, priority=13, startTime=1730992978078; duration=0sec 2024-11-07T15:22:58,214 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:58,214 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:B 2024-11-07T15:22:58,214 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#C#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:22:58,215 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/1d10e94757534602b23b251de79ec7b5 is 50, key is test_row_0/C:col10/1730992977075/Put/seqid=0 2024-11-07T15:22:58,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741926_1102 (size=12301) 2024-11-07T15:22:58,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741927_1103 (size=13289) 2024-11-07T15:22:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:22:58,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. as already flushing 2024-11-07T15:22:58,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993038354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993038357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993038362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993038363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993038363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,476 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-07T15:22:58,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,476 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:22:58,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993038471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993038471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993038471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,477 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-07T15:22:58,477 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:22:58,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993038471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993038472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T15:22:58,484 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-07T15:22:58,484 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:22:58,484 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-07T15:22:58,484 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:58,485 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:22:58,485 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:58,485 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:58,485 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:58,485 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:58,485 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:58,485 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:22:58,485 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:22:58,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:22:58,486 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:58,486 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:22:58,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:22:58,645 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/4f7d907b8e5f4b7aac3c34a39b279672 2024-11-07T15:22:58,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/165a535940324a2a83a246d185fb1b94 is 50, key is test_row_0/B:col10/1730992977121/Put/seqid=0 2024-11-07T15:22:58,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741928_1104 (size=12301) 2024-11-07T15:22:58,672 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/1d10e94757534602b23b251de79ec7b5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1d10e94757534602b23b251de79ec7b5 2024-11-07T15:22:58,679 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/C of 86b0da056c7c770742d9ba46450cf601 into 1d10e94757534602b23b251de79ec7b5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:22:58,679 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:58,679 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/C, priority=13, startTime=1730992978078; duration=0sec 2024-11-07T15:22:58,679 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:22:58,679 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:C 2024-11-07T15:22:58,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993038681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993038681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993038681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993038681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993038691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993038991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993038991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993038991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993038991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:58,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:58,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:59,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993039002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:59,060 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/165a535940324a2a83a246d185fb1b94 2024-11-07T15:22:59,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/23106ef319d846748cfc63337e792855 is 50, key is test_row_0/C:col10/1730992977121/Put/seqid=0 2024-11-07T15:22:59,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741929_1105 (size=12301) 2024-11-07T15:22:59,480 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/23106ef319d846748cfc63337e792855 2024-11-07T15:22:59,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/4f7d907b8e5f4b7aac3c34a39b279672 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/4f7d907b8e5f4b7aac3c34a39b279672 2024-11-07T15:22:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T15:22:59,494 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/4f7d907b8e5f4b7aac3c34a39b279672, entries=150, sequenceid=446, filesize=12.0 K 2024-11-07T15:22:59,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/165a535940324a2a83a246d185fb1b94 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/165a535940324a2a83a246d185fb1b94 2024-11-07T15:22:59,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51432 deadline: 1730993039502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:59,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51448 deadline: 1730993039502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:59,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51416 deadline: 1730993039502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:59,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51390 deadline: 1730993039502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:59,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:59,506 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/165a535940324a2a83a246d185fb1b94, entries=150, sequenceid=446, filesize=12.0 K 2024-11-07T15:22:59,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/23106ef319d846748cfc63337e792855 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/23106ef319d846748cfc63337e792855 2024-11-07T15:22:59,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:22:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51412 deadline: 1730993039511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:22:59,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:59,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:22:59,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:22:59,515 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/23106ef319d846748cfc63337e792855, entries=150, sequenceid=446, filesize=12.0 K 2024-11-07T15:22:59,517 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 86b0da056c7c770742d9ba46450cf601 in 1394ms, sequenceid=446, compaction requested=false 2024-11-07T15:22:59,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:22:59,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:22:59,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-07T15:22:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-07T15:22:59,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-07T15:22:59,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1750 sec 2024-11-07T15:22:59,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.1810 sec 2024-11-07T15:23:00,292 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T15:23:00,514 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:00,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T15:23:00,514 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-07T15:23:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:23:00,514 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:00,514 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:00,514 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:00,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:00,514 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-07T15:23:00,514 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-07T15:23:00,515 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:00,515 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:00,515 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:00,515 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:00,515 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:00,515 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:00,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:00,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:00,517 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-07T15:23:00,517 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:00,517 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:00,517 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:00,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:00,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/e3d88a8e60f14f6fa2c40e389a5efdac is 50, key is test_row_0/A:col10/1730992978360/Put/seqid=0 2024-11-07T15:23:00,524 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-07T15:23:00,524 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:00,524 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:00,524 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:00,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:00,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741930_1106 (size=12301) 2024-11-07T15:23:00,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/e3d88a8e60f14f6fa2c40e389a5efdac 2024-11-07T15:23:00,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/b513bfdfa4d244ba8df1d904f6d8cd72 is 50, key is test_row_0/B:col10/1730992978360/Put/seqid=0 2024-11-07T15:23:00,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741931_1107 (size=12301) 2024-11-07T15:23:00,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/b513bfdfa4d244ba8df1d904f6d8cd72 2024-11-07T15:23:00,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/93ff25e608994d1ba36d1a17b52dae34 is 50, key is test_row_0/C:col10/1730992978360/Put/seqid=0 2024-11-07T15:23:00,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741932_1108 (size=12301) 2024-11-07T15:23:01,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/93ff25e608994d1ba36d1a17b52dae34 2024-11-07T15:23:01,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/e3d88a8e60f14f6fa2c40e389a5efdac as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/e3d88a8e60f14f6fa2c40e389a5efdac 2024-11-07T15:23:01,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/e3d88a8e60f14f6fa2c40e389a5efdac, entries=150, sequenceid=473, filesize=12.0 K 2024-11-07T15:23:01,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/b513bfdfa4d244ba8df1d904f6d8cd72 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b513bfdfa4d244ba8df1d904f6d8cd72 2024-11-07T15:23:01,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b513bfdfa4d244ba8df1d904f6d8cd72, entries=150, sequenceid=473, filesize=12.0 K 2024-11-07T15:23:01,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/93ff25e608994d1ba36d1a17b52dae34 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/93ff25e608994d1ba36d1a17b52dae34 2024-11-07T15:23:01,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/93ff25e608994d1ba36d1a17b52dae34, entries=150, sequenceid=473, filesize=12.0 K 2024-11-07T15:23:01,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=6.71 KB/6870 for 86b0da056c7c770742d9ba46450cf601 in 904ms, sequenceid=473, compaction requested=true 2024-11-07T15:23:01,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:23:01,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:01,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:01,419 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:01,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:01,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:01,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86b0da056c7c770742d9ba46450cf601:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:01,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T15:23:01,419 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:01,420 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:01,420 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/A is initiating minor compaction (all files) 2024-11-07T15:23:01,420 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/A in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:23:01,420 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/a0effd952f3f4258a4618a8f3d32a716, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/4f7d907b8e5f4b7aac3c34a39b279672, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/e3d88a8e60f14f6fa2c40e389a5efdac] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=37.0 K 2024-11-07T15:23:01,420 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:01,421 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 86b0da056c7c770742d9ba46450cf601/B is initiating minor compaction (all files) 2024-11-07T15:23:01,421 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86b0da056c7c770742d9ba46450cf601/B in TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:23:01,421 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting a0effd952f3f4258a4618a8f3d32a716, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1730992976402 2024-11-07T15:23:01,421 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/d403dbf73f8d4b7284ee5e8dd8bb4968, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/165a535940324a2a83a246d185fb1b94, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b513bfdfa4d244ba8df1d904f6d8cd72] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp, totalSize=37.0 K 2024-11-07T15:23:01,421 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting d403dbf73f8d4b7284ee5e8dd8bb4968, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1730992976402 2024-11-07T15:23:01,421 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f7d907b8e5f4b7aac3c34a39b279672, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1730992977101 2024-11-07T15:23:01,422 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 165a535940324a2a83a246d185fb1b94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1730992977101 2024-11-07T15:23:01,422 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting e3d88a8e60f14f6fa2c40e389a5efdac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1730992978360 2024-11-07T15:23:01,422 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting b513bfdfa4d244ba8df1d904f6d8cd72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1730992978360 2024-11-07T15:23:01,432 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#B#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:01,433 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/93e990c89eed49f59dfd2f9a5476798a is 50, key is test_row_0/B:col10/1730992978360/Put/seqid=0 2024-11-07T15:23:01,435 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86b0da056c7c770742d9ba46450cf601#A#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:01,435 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/c503d93557bc413f8a87db3c0d1271e2 is 50, key is test_row_0/A:col10/1730992978360/Put/seqid=0 2024-11-07T15:23:01,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741933_1109 (size=13391) 2024-11-07T15:23:01,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741934_1110 (size=13391) 2024-11-07T15:23:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T15:23:01,512 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:01,512 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T15:23:01,512 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-07T15:23:01,512 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-07T15:23:01,512 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-07T15:23:01,512 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-07T15:23:01,512 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-07T15:23:01,512 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T15:23:01,512 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2757 2024-11-07T15:23:01,512 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2683 2024-11-07T15:23:01,512 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T15:23:01,513 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1279 2024-11-07T15:23:01,513 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3830 rows 2024-11-07T15:23:01,513 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1288 2024-11-07T15:23:01,513 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3864 rows 2024-11-07T15:23:01,513 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T15:23:01,513 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:23:01,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:01,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:01,513 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:01,519 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T15:23:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:01,529 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730992981529"}]},"ts":"1730992981529"} 2024-11-07T15:23:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T15:23:01,534 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T15:23:01,534 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T15:23:01,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:23:01,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86b0da056c7c770742d9ba46450cf601, UNASSIGN}] 2024-11-07T15:23:01,543 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86b0da056c7c770742d9ba46450cf601, UNASSIGN 2024-11-07T15:23:01,544 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=86b0da056c7c770742d9ba46450cf601, regionState=CLOSING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:01,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86b0da056c7c770742d9ba46450cf601, UNASSIGN because future has completed 2024-11-07T15:23:01,547 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:23:01,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:23:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T15:23:01,705 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:23:01,706 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:23:01,706 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 86b0da056c7c770742d9ba46450cf601, disabling compactions & flushes 2024-11-07T15:23:01,707 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:23:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T15:23:01,862 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/93e990c89eed49f59dfd2f9a5476798a as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/93e990c89eed49f59dfd2f9a5476798a 2024-11-07T15:23:01,863 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/c503d93557bc413f8a87db3c0d1271e2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/c503d93557bc413f8a87db3c0d1271e2 2024-11-07T15:23:01,868 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/A of 86b0da056c7c770742d9ba46450cf601 into c503d93557bc413f8a87db3c0d1271e2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:01,868 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86b0da056c7c770742d9ba46450cf601/B of 86b0da056c7c770742d9ba46450cf601 into 93e990c89eed49f59dfd2f9a5476798a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:01,868 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:23:01,868 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86b0da056c7c770742d9ba46450cf601: 2024-11-07T15:23:01,868 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:23:01,868 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/B, priority=13, startTime=1730992981419; duration=0sec 2024-11-07T15:23:01,868 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., storeName=86b0da056c7c770742d9ba46450cf601/A, priority=13, startTime=1730992981418; duration=0sec 2024-11-07T15:23:01,868 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:23:01,868 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:01,868 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. after waiting 0 ms 2024-11-07T15:23:01,868 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:01,868 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:A 2024-11-07T15:23:01,868 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:B 2024-11-07T15:23:01,868 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:23:01,869 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 86b0da056c7c770742d9ba46450cf601 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-07T15:23:01,869 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. because compaction request was cancelled 2024-11-07T15:23:01,869 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86b0da056c7c770742d9ba46450cf601:C 2024-11-07T15:23:01,872 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/40355d23e68c4379ab5459ac68c29461 is 50, key is test_row_0/A:col10/1730992980521/Put/seqid=0 2024-11-07T15:23:01,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741935_1111 (size=7415) 2024-11-07T15:23:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T15:23:02,278 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/40355d23e68c4379ab5459ac68c29461 2024-11-07T15:23:02,286 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/016ab77ce46142ab88f793129da6d946 is 50, key is test_row_0/B:col10/1730992980521/Put/seqid=0 2024-11-07T15:23:02,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741936_1112 (size=7415) 2024-11-07T15:23:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T15:23:02,695 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/016ab77ce46142ab88f793129da6d946 2024-11-07T15:23:02,702 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/59c1cf86126242439b4895c5905679d5 is 50, key is test_row_0/C:col10/1730992980521/Put/seqid=0 2024-11-07T15:23:02,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741937_1113 (size=7415) 2024-11-07T15:23:03,107 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/59c1cf86126242439b4895c5905679d5 2024-11-07T15:23:03,113 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/A/40355d23e68c4379ab5459ac68c29461 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/40355d23e68c4379ab5459ac68c29461 2024-11-07T15:23:03,120 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/40355d23e68c4379ab5459ac68c29461, entries=50, sequenceid=479, filesize=7.2 K 2024-11-07T15:23:03,121 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/B/016ab77ce46142ab88f793129da6d946 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/016ab77ce46142ab88f793129da6d946 2024-11-07T15:23:03,126 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/016ab77ce46142ab88f793129da6d946, entries=50, sequenceid=479, filesize=7.2 K 2024-11-07T15:23:03,127 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/.tmp/C/59c1cf86126242439b4895c5905679d5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/59c1cf86126242439b4895c5905679d5 2024-11-07T15:23:03,133 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/59c1cf86126242439b4895c5905679d5, entries=50, sequenceid=479, filesize=7.2 K 2024-11-07T15:23:03,134 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 86b0da056c7c770742d9ba46450cf601 in 1266ms, sequenceid=479, compaction requested=true 2024-11-07T15:23:03,134 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/3dd1233493b44b47a3500dbc9bae41ea, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/226d5ce0bb0e4669b4d064e7252e890a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/bf17493135a047e0b6513fea6c750483, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/828dc33b7b0448e0bb977d09b1c73af7, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/60231290e67345d491a1c5efbacee287, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/c53125f50c20465eb616a49f93195cb5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/6cfe8abd1fb544df85a9de4b4428ee86, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/9e2b91b62bc44198a44c59455975b6b1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/96fa6f19f383416587bcd180098e6dbf, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/3faf34ac2f364e4b8f8e1bad44c7f4bc, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/bec398fa2f93436cb820c9f537e9def6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/1341955a50384398bfd7d7e3333e7376, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/7c7e5e5fe7be4ac3afec1cdaddb4fee8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b2f92d0160274d0680ce4b6c42062865, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/7a261fb2fa4d4195a2cbee8a02600e29, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/c087ca492300478c9bc7642a1ba50124, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/2cf849e74d44456aba982dafa53df438, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/98bb0f3f26ba4ee587d0def0a666894e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/191f12fcaf654a9fb7248d29ae44cf79, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/0755b5a650e24f77afd9de75a86d029e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/371be59ec416441f86117ef2952a0e76, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b148e4ce2eeb452fa77c51a66efc890f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/e9de5cf9dd3c4df598bfb5b10cd22574, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/db8797a21b7140ca9e656c377ec3b959, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b55f98a912964fd0bb2cfbff7abb68b8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/404f9a3802264884abe45edd529399fd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/49f26e848ab44ce0b097b04852d324cc, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/466eba214d4d41eb8ac3c9b3452920d2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/a0effd952f3f4258a4618a8f3d32a716, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/a93a74f766d14213b78da1f4428346cd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/4f7d907b8e5f4b7aac3c34a39b279672, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/e3d88a8e60f14f6fa2c40e389a5efdac] to archive 2024-11-07T15:23:03,138 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:23:03,143 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/3dd1233493b44b47a3500dbc9bae41ea to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/3dd1233493b44b47a3500dbc9bae41ea 2024-11-07T15:23:03,145 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/226d5ce0bb0e4669b4d064e7252e890a to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/226d5ce0bb0e4669b4d064e7252e890a 2024-11-07T15:23:03,147 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/bf17493135a047e0b6513fea6c750483 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/bf17493135a047e0b6513fea6c750483 2024-11-07T15:23:03,148 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/828dc33b7b0448e0bb977d09b1c73af7 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/828dc33b7b0448e0bb977d09b1c73af7 2024-11-07T15:23:03,149 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/60231290e67345d491a1c5efbacee287 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/60231290e67345d491a1c5efbacee287 2024-11-07T15:23:03,151 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/c53125f50c20465eb616a49f93195cb5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/c53125f50c20465eb616a49f93195cb5 2024-11-07T15:23:03,152 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/6cfe8abd1fb544df85a9de4b4428ee86 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/6cfe8abd1fb544df85a9de4b4428ee86 2024-11-07T15:23:03,154 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/9e2b91b62bc44198a44c59455975b6b1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/9e2b91b62bc44198a44c59455975b6b1 2024-11-07T15:23:03,155 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/96fa6f19f383416587bcd180098e6dbf to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/96fa6f19f383416587bcd180098e6dbf 2024-11-07T15:23:03,156 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/3faf34ac2f364e4b8f8e1bad44c7f4bc to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/3faf34ac2f364e4b8f8e1bad44c7f4bc 2024-11-07T15:23:03,158 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/bec398fa2f93436cb820c9f537e9def6 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/bec398fa2f93436cb820c9f537e9def6 2024-11-07T15:23:03,159 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/1341955a50384398bfd7d7e3333e7376 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/1341955a50384398bfd7d7e3333e7376 2024-11-07T15:23:03,161 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/7c7e5e5fe7be4ac3afec1cdaddb4fee8 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/7c7e5e5fe7be4ac3afec1cdaddb4fee8 2024-11-07T15:23:03,163 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b2f92d0160274d0680ce4b6c42062865 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b2f92d0160274d0680ce4b6c42062865 2024-11-07T15:23:03,165 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/7a261fb2fa4d4195a2cbee8a02600e29 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/7a261fb2fa4d4195a2cbee8a02600e29 2024-11-07T15:23:03,166 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/c087ca492300478c9bc7642a1ba50124 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/c087ca492300478c9bc7642a1ba50124 2024-11-07T15:23:03,168 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/2cf849e74d44456aba982dafa53df438 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/2cf849e74d44456aba982dafa53df438 2024-11-07T15:23:03,169 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/98bb0f3f26ba4ee587d0def0a666894e to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/98bb0f3f26ba4ee587d0def0a666894e 2024-11-07T15:23:03,170 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/191f12fcaf654a9fb7248d29ae44cf79 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/191f12fcaf654a9fb7248d29ae44cf79 2024-11-07T15:23:03,172 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/0755b5a650e24f77afd9de75a86d029e to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/0755b5a650e24f77afd9de75a86d029e 2024-11-07T15:23:03,173 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/371be59ec416441f86117ef2952a0e76 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/371be59ec416441f86117ef2952a0e76 2024-11-07T15:23:03,174 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b148e4ce2eeb452fa77c51a66efc890f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b148e4ce2eeb452fa77c51a66efc890f 2024-11-07T15:23:03,176 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/e9de5cf9dd3c4df598bfb5b10cd22574 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/e9de5cf9dd3c4df598bfb5b10cd22574 2024-11-07T15:23:03,177 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/db8797a21b7140ca9e656c377ec3b959 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/db8797a21b7140ca9e656c377ec3b959 2024-11-07T15:23:03,178 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b55f98a912964fd0bb2cfbff7abb68b8 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/b55f98a912964fd0bb2cfbff7abb68b8 2024-11-07T15:23:03,179 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/404f9a3802264884abe45edd529399fd to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/404f9a3802264884abe45edd529399fd 2024-11-07T15:23:03,181 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/49f26e848ab44ce0b097b04852d324cc to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/49f26e848ab44ce0b097b04852d324cc 2024-11-07T15:23:03,182 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/466eba214d4d41eb8ac3c9b3452920d2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/466eba214d4d41eb8ac3c9b3452920d2 2024-11-07T15:23:03,183 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/a0effd952f3f4258a4618a8f3d32a716 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/a0effd952f3f4258a4618a8f3d32a716 2024-11-07T15:23:03,185 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/a93a74f766d14213b78da1f4428346cd to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/a93a74f766d14213b78da1f4428346cd 2024-11-07T15:23:03,186 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/4f7d907b8e5f4b7aac3c34a39b279672 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/4f7d907b8e5f4b7aac3c34a39b279672 2024-11-07T15:23:03,188 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/e3d88a8e60f14f6fa2c40e389a5efdac to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/e3d88a8e60f14f6fa2c40e389a5efdac 2024-11-07T15:23:03,202 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/20f448f38e4641e8bff89ef6bf7f8962, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/28908b548ef64b9da9fff0f7d9074ea7, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/dbfeeaa9967e4e58aa6f1525b2118cb8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8bb9f41c0c7940a0922dc1783537e3f2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/92b1edcba0a14ab59f13156f79720a76, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/54f1f3025a194b51a1da46ae7519c341, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/93b5fba9e47d4f7ab5799a9d27beeb12, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/a3d8d90f1d5c41bba0823d562be28524, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/bb39c73020d54ab89e1a8b8ffb333ad4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b0485c9ef95741d9a93ae6d73fa2aa93, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8879e2d5ca044cd0a9498bcc5e2db4fc, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/7c53de08047e4d83b80a8a6c8b521996, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/13733ecd3e2b46ad852fe911ba208ab0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/29a3304aede3432399e6678794819ed4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/5e14dc22ded548b2b4a0cf35e3e1686c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/246b2c028e564690b3cc38ca4442bc25, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/04c31c4948b94cec9ced08c4d0363324, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/4d8056adda0941e098463fad4a962efb, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b39316d32462476493c253fa6b70e566, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8b557a1316d34be6a3fe13ada5f55ca7, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/94c36ce7ddc54505b604cbbef2484ecc, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/faaf9033735f41d198fc1b5300e2c4a1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/6863b5d230db4759b0495c832a338b05, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/d4e7d723bb9d46dc88cfde240afbcf49, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/5349722ed979470b9963d7201411ed26, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/356ca9b168ec42d7b80af7c6745c1483, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/f548367c027842b6adcded547bddf756, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b186ad9e49b94fb49eb291765ec819d6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/d403dbf73f8d4b7284ee5e8dd8bb4968, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/4536402980c1408983d6e0a7be35b1c4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/165a535940324a2a83a246d185fb1b94, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b513bfdfa4d244ba8df1d904f6d8cd72] to archive 2024-11-07T15:23:03,203 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:23:03,205 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/20f448f38e4641e8bff89ef6bf7f8962 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/20f448f38e4641e8bff89ef6bf7f8962 2024-11-07T15:23:03,206 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/28908b548ef64b9da9fff0f7d9074ea7 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/28908b548ef64b9da9fff0f7d9074ea7 2024-11-07T15:23:03,207 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/dbfeeaa9967e4e58aa6f1525b2118cb8 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/dbfeeaa9967e4e58aa6f1525b2118cb8 2024-11-07T15:23:03,209 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8bb9f41c0c7940a0922dc1783537e3f2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8bb9f41c0c7940a0922dc1783537e3f2 2024-11-07T15:23:03,210 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/92b1edcba0a14ab59f13156f79720a76 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/92b1edcba0a14ab59f13156f79720a76 2024-11-07T15:23:03,211 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/54f1f3025a194b51a1da46ae7519c341 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/54f1f3025a194b51a1da46ae7519c341 2024-11-07T15:23:03,212 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/93b5fba9e47d4f7ab5799a9d27beeb12 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/93b5fba9e47d4f7ab5799a9d27beeb12 2024-11-07T15:23:03,214 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/a3d8d90f1d5c41bba0823d562be28524 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/a3d8d90f1d5c41bba0823d562be28524 2024-11-07T15:23:03,215 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/bb39c73020d54ab89e1a8b8ffb333ad4 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/bb39c73020d54ab89e1a8b8ffb333ad4 2024-11-07T15:23:03,216 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b0485c9ef95741d9a93ae6d73fa2aa93 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b0485c9ef95741d9a93ae6d73fa2aa93 2024-11-07T15:23:03,217 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8879e2d5ca044cd0a9498bcc5e2db4fc to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8879e2d5ca044cd0a9498bcc5e2db4fc 2024-11-07T15:23:03,218 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/7c53de08047e4d83b80a8a6c8b521996 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/7c53de08047e4d83b80a8a6c8b521996 2024-11-07T15:23:03,219 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/13733ecd3e2b46ad852fe911ba208ab0 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/13733ecd3e2b46ad852fe911ba208ab0 2024-11-07T15:23:03,220 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/29a3304aede3432399e6678794819ed4 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/29a3304aede3432399e6678794819ed4 2024-11-07T15:23:03,222 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/5e14dc22ded548b2b4a0cf35e3e1686c to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/5e14dc22ded548b2b4a0cf35e3e1686c 2024-11-07T15:23:03,223 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/246b2c028e564690b3cc38ca4442bc25 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/246b2c028e564690b3cc38ca4442bc25 2024-11-07T15:23:03,224 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/04c31c4948b94cec9ced08c4d0363324 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/04c31c4948b94cec9ced08c4d0363324 2024-11-07T15:23:03,225 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/4d8056adda0941e098463fad4a962efb to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/4d8056adda0941e098463fad4a962efb 2024-11-07T15:23:03,226 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b39316d32462476493c253fa6b70e566 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b39316d32462476493c253fa6b70e566 2024-11-07T15:23:03,228 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8b557a1316d34be6a3fe13ada5f55ca7 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/8b557a1316d34be6a3fe13ada5f55ca7 2024-11-07T15:23:03,229 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/94c36ce7ddc54505b604cbbef2484ecc to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/94c36ce7ddc54505b604cbbef2484ecc 2024-11-07T15:23:03,230 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/faaf9033735f41d198fc1b5300e2c4a1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/faaf9033735f41d198fc1b5300e2c4a1 2024-11-07T15:23:03,231 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/6863b5d230db4759b0495c832a338b05 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/6863b5d230db4759b0495c832a338b05 2024-11-07T15:23:03,232 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/d4e7d723bb9d46dc88cfde240afbcf49 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/d4e7d723bb9d46dc88cfde240afbcf49 2024-11-07T15:23:03,233 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/5349722ed979470b9963d7201411ed26 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/5349722ed979470b9963d7201411ed26 2024-11-07T15:23:03,234 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/356ca9b168ec42d7b80af7c6745c1483 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/356ca9b168ec42d7b80af7c6745c1483 2024-11-07T15:23:03,235 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/f548367c027842b6adcded547bddf756 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/f548367c027842b6adcded547bddf756 2024-11-07T15:23:03,236 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b186ad9e49b94fb49eb291765ec819d6 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b186ad9e49b94fb49eb291765ec819d6 2024-11-07T15:23:03,238 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/d403dbf73f8d4b7284ee5e8dd8bb4968 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/d403dbf73f8d4b7284ee5e8dd8bb4968 2024-11-07T15:23:03,239 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/4536402980c1408983d6e0a7be35b1c4 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/4536402980c1408983d6e0a7be35b1c4 2024-11-07T15:23:03,240 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/165a535940324a2a83a246d185fb1b94 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/165a535940324a2a83a246d185fb1b94 2024-11-07T15:23:03,242 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b513bfdfa4d244ba8df1d904f6d8cd72 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/b513bfdfa4d244ba8df1d904f6d8cd72 2024-11-07T15:23:03,243 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b9b3502525f6446db9015e8a19a5a6f3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b27cf54d98ef4d9d8cfc5a69c9ef676f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/4f56cb5d1eff41b89a5d413092c9f3c9, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/55e4d5939b504b649c4584e2a0fc2e36, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/615604929ad04eeca86e5c8ca00ff195, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/172f47523ebd4495bfed6cf46e4520e3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/5db684d7ea4847938d6d8d9edd79d057, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/6d42596a070c4b3596071655809861cf, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/71e26f5da3bd4f12894132735853b131, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1d856d1166664eb8b78525abf20ea95b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/90ab5ab273814cf694819521640de4d6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/a34bb7b891534b868e86c1ff7b348a51, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/8e3c81a950274dcb87c9df8956f12c20, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/880e5a30f8fd48e6ac652f972c48ce16, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1edbba34c34b4c5a93cb40d860f072e7, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/748fcc1cb6b54c1580a923c31e991959, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/53d3317546e84d779f3d3331e509811d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/dfe74fcdef9d4ef9a5f1def7776831d2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/9e8aae2433db4f1ea37c0b548c2c63c7, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/d518f060439b4548abf228b0b01ba481, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/4198544e0ed140c682baf33f950d9c62, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/54f42d03e00b44b69a9dc83d260968a3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/6235ceffaba646449e759a4199f0110d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1a4dd5963b6b4ab7b701622b0e523f71, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b19b472fec984c3ea6af58a8efc65d89, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/7e6ce7d4739246cab3591951020a8af2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/c6313bc25fb24dc78e67fe61d256d8a6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/d6d93a2cc3f64495b5134dfdf184b549, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/bd8913271ade420db14f9e3fb7740cff] to archive 2024-11-07T15:23:03,244 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:23:03,246 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b9b3502525f6446db9015e8a19a5a6f3 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b9b3502525f6446db9015e8a19a5a6f3 2024-11-07T15:23:03,247 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b27cf54d98ef4d9d8cfc5a69c9ef676f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b27cf54d98ef4d9d8cfc5a69c9ef676f 2024-11-07T15:23:03,248 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/4f56cb5d1eff41b89a5d413092c9f3c9 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/4f56cb5d1eff41b89a5d413092c9f3c9 2024-11-07T15:23:03,249 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/55e4d5939b504b649c4584e2a0fc2e36 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/55e4d5939b504b649c4584e2a0fc2e36 2024-11-07T15:23:03,251 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/615604929ad04eeca86e5c8ca00ff195 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/615604929ad04eeca86e5c8ca00ff195 2024-11-07T15:23:03,252 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/172f47523ebd4495bfed6cf46e4520e3 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/172f47523ebd4495bfed6cf46e4520e3 2024-11-07T15:23:03,253 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/5db684d7ea4847938d6d8d9edd79d057 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/5db684d7ea4847938d6d8d9edd79d057 2024-11-07T15:23:03,254 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/6d42596a070c4b3596071655809861cf to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/6d42596a070c4b3596071655809861cf 2024-11-07T15:23:03,255 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/71e26f5da3bd4f12894132735853b131 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/71e26f5da3bd4f12894132735853b131 2024-11-07T15:23:03,256 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1d856d1166664eb8b78525abf20ea95b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1d856d1166664eb8b78525abf20ea95b 2024-11-07T15:23:03,257 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/90ab5ab273814cf694819521640de4d6 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/90ab5ab273814cf694819521640de4d6 2024-11-07T15:23:03,258 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/a34bb7b891534b868e86c1ff7b348a51 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/a34bb7b891534b868e86c1ff7b348a51 2024-11-07T15:23:03,259 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/8e3c81a950274dcb87c9df8956f12c20 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/8e3c81a950274dcb87c9df8956f12c20 2024-11-07T15:23:03,260 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/880e5a30f8fd48e6ac652f972c48ce16 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/880e5a30f8fd48e6ac652f972c48ce16 2024-11-07T15:23:03,262 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1edbba34c34b4c5a93cb40d860f072e7 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1edbba34c34b4c5a93cb40d860f072e7 2024-11-07T15:23:03,263 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/748fcc1cb6b54c1580a923c31e991959 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/748fcc1cb6b54c1580a923c31e991959 2024-11-07T15:23:03,264 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/53d3317546e84d779f3d3331e509811d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/53d3317546e84d779f3d3331e509811d 2024-11-07T15:23:03,265 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/dfe74fcdef9d4ef9a5f1def7776831d2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/dfe74fcdef9d4ef9a5f1def7776831d2 2024-11-07T15:23:03,266 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/9e8aae2433db4f1ea37c0b548c2c63c7 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/9e8aae2433db4f1ea37c0b548c2c63c7 2024-11-07T15:23:03,267 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/d518f060439b4548abf228b0b01ba481 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/d518f060439b4548abf228b0b01ba481 2024-11-07T15:23:03,269 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/4198544e0ed140c682baf33f950d9c62 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/4198544e0ed140c682baf33f950d9c62 2024-11-07T15:23:03,270 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/54f42d03e00b44b69a9dc83d260968a3 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/54f42d03e00b44b69a9dc83d260968a3 2024-11-07T15:23:03,271 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/6235ceffaba646449e759a4199f0110d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/6235ceffaba646449e759a4199f0110d 2024-11-07T15:23:03,273 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1a4dd5963b6b4ab7b701622b0e523f71 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1a4dd5963b6b4ab7b701622b0e523f71 2024-11-07T15:23:03,274 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b19b472fec984c3ea6af58a8efc65d89 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/b19b472fec984c3ea6af58a8efc65d89 2024-11-07T15:23:03,275 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/7e6ce7d4739246cab3591951020a8af2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/7e6ce7d4739246cab3591951020a8af2 2024-11-07T15:23:03,277 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/c6313bc25fb24dc78e67fe61d256d8a6 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/c6313bc25fb24dc78e67fe61d256d8a6 2024-11-07T15:23:03,278 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/d6d93a2cc3f64495b5134dfdf184b549 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/d6d93a2cc3f64495b5134dfdf184b549 2024-11-07T15:23:03,279 DEBUG [StoreCloser-TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/bd8913271ade420db14f9e3fb7740cff to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/bd8913271ade420db14f9e3fb7740cff 2024-11-07T15:23:03,284 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/recovered.edits/482.seqid, newMaxSeqId=482, maxSeqId=1 2024-11-07T15:23:03,285 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. 2024-11-07T15:23:03,285 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 86b0da056c7c770742d9ba46450cf601: Waiting for close lock at 1730992981706Running coprocessor pre-close hooks at 1730992981706Disabling compacts and flushes for region at 1730992981706Disabling writes for close at 1730992981868 (+162 ms)Obtaining lock to block concurrent updates at 1730992981869 (+1 ms)Preparing flush snapshotting stores in 86b0da056c7c770742d9ba46450cf601 at 1730992981869Finished memstore snapshotting TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1730992981869Flushing stores of TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601. at 1730992981869Flushing 86b0da056c7c770742d9ba46450cf601/A: creating writer at 1730992981869Flushing 86b0da056c7c770742d9ba46450cf601/A: appending metadata at 1730992981872 (+3 ms)Flushing 86b0da056c7c770742d9ba46450cf601/A: closing flushed file at 1730992981872Flushing 86b0da056c7c770742d9ba46450cf601/B: creating writer at 1730992982283 (+411 ms)Flushing 86b0da056c7c770742d9ba46450cf601/B: appending metadata at 1730992982286 (+3 ms)Flushing 86b0da056c7c770742d9ba46450cf601/B: closing flushed file at 1730992982286Flushing 86b0da056c7c770742d9ba46450cf601/C: creating writer at 1730992982699 (+413 ms)Flushing 86b0da056c7c770742d9ba46450cf601/C: appending metadata at 1730992982702 (+3 ms)Flushing 86b0da056c7c770742d9ba46450cf601/C: closing flushed file at 1730992982702Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37b3bf4d: reopening flushed file at 1730992983112 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47f9ce52: reopening flushed file at 1730992983120 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@306b3ad4: reopening flushed file at 1730992983127 (+7 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 86b0da056c7c770742d9ba46450cf601 in 1266ms, sequenceid=479, compaction requested=true at 1730992983134 (+7 ms)Writing region close event to WAL at 1730992983280 (+146 ms)Running coprocessor post-close hooks at 1730992983285 (+5 ms)Closed at 1730992983285 2024-11-07T15:23:03,288 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 86b0da056c7c770742d9ba46450cf601 2024-11-07T15:23:03,288 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=86b0da056c7c770742d9ba46450cf601, regionState=CLOSED 2024-11-07T15:23:03,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:23:03,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-07T15:23:03,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 86b0da056c7c770742d9ba46450cf601, server=7b3e4e8e12ef,33849,1730992951906 in 1.7450 sec 2024-11-07T15:23:03,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-07T15:23:03,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86b0da056c7c770742d9ba46450cf601, UNASSIGN in 1.7520 sec 2024-11-07T15:23:03,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-07T15:23:03,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7600 sec 2024-11-07T15:23:03,300 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730992983300"}]},"ts":"1730992983300"} 2024-11-07T15:23:03,305 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T15:23:03,305 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T15:23:03,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7850 sec 2024-11-07T15:23:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T15:23:03,681 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:03,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T15:23:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:03,693 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-07T15:23:03,695 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:03,699 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601 2024-11-07T15:23:03,703 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/recovered.edits] 2024-11-07T15:23:03,706 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/40355d23e68c4379ab5459ac68c29461 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/40355d23e68c4379ab5459ac68c29461 2024-11-07T15:23:03,707 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/c503d93557bc413f8a87db3c0d1271e2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/A/c503d93557bc413f8a87db3c0d1271e2 2024-11-07T15:23:03,710 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/016ab77ce46142ab88f793129da6d946 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/016ab77ce46142ab88f793129da6d946 2024-11-07T15:23:03,712 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/93e990c89eed49f59dfd2f9a5476798a to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/B/93e990c89eed49f59dfd2f9a5476798a 2024-11-07T15:23:03,715 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1d10e94757534602b23b251de79ec7b5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/1d10e94757534602b23b251de79ec7b5 2024-11-07T15:23:03,716 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/23106ef319d846748cfc63337e792855 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/23106ef319d846748cfc63337e792855 2024-11-07T15:23:03,718 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/59c1cf86126242439b4895c5905679d5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/59c1cf86126242439b4895c5905679d5 2024-11-07T15:23:03,720 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/93ff25e608994d1ba36d1a17b52dae34 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/C/93ff25e608994d1ba36d1a17b52dae34 2024-11-07T15:23:03,723 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/recovered.edits/482.seqid to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601/recovered.edits/482.seqid 2024-11-07T15:23:03,724 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/86b0da056c7c770742d9ba46450cf601 2024-11-07T15:23:03,724 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T15:23:03,731 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:03,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-07T15:23:03,740 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T15:23:03,765 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T15:23:03,767 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:03,767 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T15:23:03,767 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730992983767"}]},"ts":"9223372036854775807"} 2024-11-07T15:23:03,771 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T15:23:03,771 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 86b0da056c7c770742d9ba46450cf601, NAME => 'TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T15:23:03,772 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T15:23:03,772 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730992983772"}]},"ts":"9223372036854775807"} 2024-11-07T15:23:03,775 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T15:23:03,776 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:03,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 90 msec 2024-11-07T15:23:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-07T15:23:03,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T15:23:03,802 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:03,819 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=240 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1266856539_22 at /127.0.0.1:52720 [Waiting for operation #290] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1266856539_22 at /127.0.0.1:42962 [Waiting for operation #64] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;7b3e4e8e12ef:33849-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1165652502_22 at /127.0.0.1:43200 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1165652502_22 at /127.0.0.1:52908 [Waiting for operation #271] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=474 (was 389) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1635 (was 2871) 2024-11-07T15:23:03,827 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=240, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=474, ProcessCount=11, AvailableMemoryMB=1635 2024-11-07T15:23:03,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:23:03,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:23:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:03,830 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T15:23:03,831 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:03,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-07T15:23:03,831 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T15:23:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T15:23:03,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741938_1114 (size=959) 2024-11-07T15:23:03,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T15:23:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T15:23:04,239 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938 2024-11-07T15:23:04,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741939_1115 (size=53) 2024-11-07T15:23:04,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T15:23:04,646 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:23:04,646 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 518e68065b228feb1bbc2642f1a6f79d, disabling compactions & flushes 2024-11-07T15:23:04,646 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:04,646 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:04,646 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. after waiting 0 ms 2024-11-07T15:23:04,646 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:04,646 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:04,646 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 518e68065b228feb1bbc2642f1a6f79d: Waiting for close lock at 1730992984646Disabling compacts and flushes for region at 1730992984646Disabling writes for close at 1730992984646Writing region close event to WAL at 1730992984646Closed at 1730992984646 2024-11-07T15:23:04,648 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T15:23:04,648 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730992984648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730992984648"}]},"ts":"1730992984648"} 2024-11-07T15:23:04,650 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T15:23:04,651 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T15:23:04,651 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730992984651"}]},"ts":"1730992984651"} 2024-11-07T15:23:04,652 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T15:23:04,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=518e68065b228feb1bbc2642f1a6f79d, ASSIGN}] 2024-11-07T15:23:04,654 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=518e68065b228feb1bbc2642f1a6f79d, ASSIGN 2024-11-07T15:23:04,654 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=518e68065b228feb1bbc2642f1a6f79d, ASSIGN; state=OFFLINE, location=7b3e4e8e12ef,33849,1730992951906; forceNewPlan=false, retain=false 2024-11-07T15:23:04,805 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=518e68065b228feb1bbc2642f1a6f79d, regionState=OPENING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:04,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=518e68065b228feb1bbc2642f1a6f79d, ASSIGN because future has completed 2024-11-07T15:23:04,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:23:04,968 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:04,969 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:23:04,969 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:04,969 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:23:04,970 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:04,970 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T15:23:04,972 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:04,976 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 518e68065b228feb1bbc2642f1a6f79d columnFamilyName A 2024-11-07T15:23:04,976 DEBUG [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:04,977 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] regionserver.HStore(327): Store=518e68065b228feb1bbc2642f1a6f79d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:23:04,977 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:04,978 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 518e68065b228feb1bbc2642f1a6f79d columnFamilyName B 2024-11-07T15:23:04,978 DEBUG [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:04,978 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] regionserver.HStore(327): Store=518e68065b228feb1bbc2642f1a6f79d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:23:04,979 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:04,980 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 518e68065b228feb1bbc2642f1a6f79d columnFamilyName C 2024-11-07T15:23:04,980 DEBUG [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:04,980 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] regionserver.HStore(327): Store=518e68065b228feb1bbc2642f1a6f79d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:23:04,980 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:04,981 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:04,981 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:04,982 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:04,982 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:04,983 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:23:04,984 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:04,986 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T15:23:04,986 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 518e68065b228feb1bbc2642f1a6f79d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61024957, jitterRate=-0.09065727889537811}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:23:04,986 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:04,987 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 518e68065b228feb1bbc2642f1a6f79d: Running coprocessor pre-open hook at 1730992984970Writing region info on filesystem at 1730992984970Initializing all the Stores at 1730992984971 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730992984971Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730992984972 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730992984972Cleaning up temporary data from old regions at 1730992984982 (+10 ms)Running coprocessor post-open hooks at 1730992984986 (+4 ms)Region opened successfully at 1730992984987 (+1 ms) 2024-11-07T15:23:04,988 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., pid=34, masterSystemTime=1730992984963 2024-11-07T15:23:04,990 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:04,990 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:04,990 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=518e68065b228feb1bbc2642f1a6f79d, regionState=OPEN, openSeqNum=2, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:04,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:23:04,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-07T15:23:04,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 in 183 msec 2024-11-07T15:23:04,997 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-07T15:23:04,997 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=518e68065b228feb1bbc2642f1a6f79d, ASSIGN in 342 msec 2024-11-07T15:23:04,998 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T15:23:04,998 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730992984998"}]},"ts":"1730992984998"} 2024-11-07T15:23:05,000 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T15:23:05,001 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T15:23:05,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-11-07T15:23:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-07T15:23:05,982 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:05,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c21cd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:05,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:05,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:05,988 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c376f61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:05,992 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2766031b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:05,994 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:05,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:06,000 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T15:23:06,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b3e4e8e12ef,46811,1730992951165 2024-11-07T15:23:06,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b72cf43 2024-11-07T15:23:06,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T15:23:06,002 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T15:23:06,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:23:06,009 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:23:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:06,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741940_1116 (size=995) 2024-11-07T15:23:06,435 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T15:23:06,435 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T15:23:06,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:23:06,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=518e68065b228feb1bbc2642f1a6f79d, REOPEN/MOVE}] 2024-11-07T15:23:06,451 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=518e68065b228feb1bbc2642f1a6f79d, REOPEN/MOVE 2024-11-07T15:23:06,451 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=518e68065b228feb1bbc2642f1a6f79d, regionState=CLOSING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:06,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=518e68065b228feb1bbc2642f1a6f79d, REOPEN/MOVE because future has completed 2024-11-07T15:23:06,453 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:23:06,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:23:06,606 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:06,606 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:23:06,606 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 518e68065b228feb1bbc2642f1a6f79d, disabling compactions & flushes 2024-11-07T15:23:06,606 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:06,606 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:06,606 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. after waiting 0 ms 2024-11-07T15:23:06,606 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:06,613 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T15:23:06,615 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:06,615 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 518e68065b228feb1bbc2642f1a6f79d: Waiting for close lock at 1730992986606Running coprocessor pre-close hooks at 1730992986606Disabling compacts and flushes for region at 1730992986606Disabling writes for close at 1730992986606Writing region close event to WAL at 1730992986607 (+1 ms)Running coprocessor post-close hooks at 1730992986614 (+7 ms)Closed at 1730992986615 (+1 ms) 2024-11-07T15:23:06,615 WARN [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 518e68065b228feb1bbc2642f1a6f79d to self. 2024-11-07T15:23:06,619 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:06,620 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=518e68065b228feb1bbc2642f1a6f79d, regionState=CLOSED 2024-11-07T15:23:06,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:23:06,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-07T15:23:06,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 in 171 msec 2024-11-07T15:23:06,627 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=518e68065b228feb1bbc2642f1a6f79d, REOPEN/MOVE; state=CLOSED, location=7b3e4e8e12ef,33849,1730992951906; forceNewPlan=false, retain=true 2024-11-07T15:23:06,777 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=518e68065b228feb1bbc2642f1a6f79d, regionState=OPENING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:06,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=518e68065b228feb1bbc2642f1a6f79d, REOPEN/MOVE because future has completed 2024-11-07T15:23:06,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:23:06,936 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:06,936 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:23:06,937 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:06,937 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:23:06,937 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:06,937 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:06,941 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:06,949 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 518e68065b228feb1bbc2642f1a6f79d columnFamilyName A 2024-11-07T15:23:06,951 DEBUG [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:06,952 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] regionserver.HStore(327): Store=518e68065b228feb1bbc2642f1a6f79d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:23:06,953 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:06,953 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 518e68065b228feb1bbc2642f1a6f79d columnFamilyName B 2024-11-07T15:23:06,953 DEBUG [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:06,954 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] regionserver.HStore(327): Store=518e68065b228feb1bbc2642f1a6f79d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:23:06,954 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:06,955 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 518e68065b228feb1bbc2642f1a6f79d columnFamilyName C 2024-11-07T15:23:06,955 DEBUG [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:06,955 INFO [StoreOpener-518e68065b228feb1bbc2642f1a6f79d-1 {}] regionserver.HStore(327): Store=518e68065b228feb1bbc2642f1a6f79d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:23:06,955 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:06,956 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:06,957 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:06,958 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:06,958 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:06,959 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:23:06,960 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:06,961 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 518e68065b228feb1bbc2642f1a6f79d; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62001149, jitterRate=-0.07611088454723358}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:23:06,961 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:06,962 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 518e68065b228feb1bbc2642f1a6f79d: Running coprocessor pre-open hook at 1730992986937Writing region info on filesystem at 1730992986937Initializing all the Stores at 1730992986939 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730992986939Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730992986939Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730992986939Cleaning up temporary data from old regions at 1730992986958 (+19 ms)Running coprocessor post-open hooks at 1730992986961 (+3 ms)Region opened successfully at 1730992986962 (+1 ms) 2024-11-07T15:23:06,963 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., pid=39, masterSystemTime=1730992986931 2024-11-07T15:23:06,965 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:06,965 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:06,965 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=518e68065b228feb1bbc2642f1a6f79d, regionState=OPEN, openSeqNum=5, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:06,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:23:06,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-07T15:23:06,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 in 188 msec 2024-11-07T15:23:06,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-07T15:23:06,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=518e68065b228feb1bbc2642f1a6f79d, REOPEN/MOVE in 519 msec 2024-11-07T15:23:06,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-07T15:23:06,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 531 msec 2024-11-07T15:23:06,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 963 msec 2024-11-07T15:23:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-07T15:23:06,979 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:06,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b474cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:06,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:06,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:06,981 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@448b8918, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:06,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:06,982 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:06,982 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:06,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1745a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:06,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d2c57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:06,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:06,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:06,985 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:06,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:06,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:06,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7405a174, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:06,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:06,987 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd31c47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:06,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bc37772, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:06,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:06,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:06,990 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:06,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:06,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:06,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b42226f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:06,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:06,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:06,991 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:06,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ece8381, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:06,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1987d108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:06,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:06,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:06,994 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ff16adb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:06,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:06,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:06,996 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@362ccb5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:06,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5fd39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:06,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:06,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:06,998 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:06,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:06,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:06,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66fd648c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:06,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:06,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:06,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:06,999 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:07,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@707939c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:07,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f7e914, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:07,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:07,001 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:07,002 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f88284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:07,003 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:07,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69088e73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:07,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@559060c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:07,004 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:07,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:07,005 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:07,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ddb9398, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:07,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:07,007 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:07,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3034141e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:07,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@405710cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:07,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:07,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:07,009 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4938e2ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:07,011 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:07,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10a94543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:07,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@769181eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:07,012 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:07,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:07,013 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32401964, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:07,015 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8288ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:07,016 DEBUG [Thread-596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:07,016 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:07,016 DEBUG [Thread-613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:07,016 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:07,017 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:07,017 DEBUG [Thread-609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:07,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:07,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:07,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:07,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:07,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:07,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:07,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:07,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:07,019 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:07,020 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:07,020 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:07,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:07,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:07,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-07T15:23:07,022 DEBUG [Thread-604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T15:23:07,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:23:07,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:23:07,024 DEBUG [Thread-598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:07,024 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:07,025 DEBUG [Thread-607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:07,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:23:07,026 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:07,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:23:07,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:07,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:23:07,027 DEBUG [Thread-607 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:07,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:07,027 DEBUG [Thread-607 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:07,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:07,028 DEBUG [Thread-604 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:07,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:07,029 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:07,029 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:07,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:23:07,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:23:07,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:23:07,032 DEBUG [Thread-604 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:23:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:07,042 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:07,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:23:07,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110712e818d7c87b4160929fe707e52e50e7_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730992987037/Put/seqid=0 2024-11-07T15:23:07,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741941_1117 (size=14594) 2024-11-07T15:23:07,107 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:07,119 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110712e818d7c87b4160929fe707e52e50e7_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110712e818d7c87b4160929fe707e52e50e7_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:07,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993047103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993047106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,121 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/133ad1657bce4ee9b05c2eda97eb220e, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:07,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993047108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993047114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/133ad1657bce4ee9b05c2eda97eb220e is 175, key is test_row_0/A:col10/1730992987037/Put/seqid=0 2024-11-07T15:23:07,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993047119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T15:23:07,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741942_1118 (size=39549) 2024-11-07T15:23:07,147 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/133ad1657bce4ee9b05c2eda97eb220e 2024-11-07T15:23:07,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T15:23:07,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/d9c5a66458974763b6889095e73f882f is 50, key is test_row_0/B:col10/1730992987037/Put/seqid=0 2024-11-07T15:23:07,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:07,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:07,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:07,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741943_1119 (size=12001) 2024-11-07T15:23:07,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/d9c5a66458974763b6889095e73f882f 2024-11-07T15:23:07,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/152a520d914a48248da9b02121803000 is 50, key is test_row_0/C:col10/1730992987037/Put/seqid=0 2024-11-07T15:23:07,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993047231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993047231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993047232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993047234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741944_1120 (size=12001) 2024-11-07T15:23:07,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993047241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T15:23:07,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:07,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:07,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:07,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T15:23:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993047451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993047451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993047452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993047452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993047461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T15:23:07,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:07,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:07,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:07,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/152a520d914a48248da9b02121803000 2024-11-07T15:23:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T15:23:07,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T15:23:07,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:07,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:07,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:07,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/133ad1657bce4ee9b05c2eda97eb220e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/133ad1657bce4ee9b05c2eda97eb220e 2024-11-07T15:23:07,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/133ad1657bce4ee9b05c2eda97eb220e, entries=200, sequenceid=15, filesize=38.6 K 2024-11-07T15:23:07,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/d9c5a66458974763b6889095e73f882f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/d9c5a66458974763b6889095e73f882f 2024-11-07T15:23:07,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/d9c5a66458974763b6889095e73f882f, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T15:23:07,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/152a520d914a48248da9b02121803000 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/152a520d914a48248da9b02121803000 2024-11-07T15:23:07,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/152a520d914a48248da9b02121803000, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T15:23:07,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 518e68065b228feb1bbc2642f1a6f79d in 650ms, sequenceid=15, compaction requested=false 2024-11-07T15:23:07,692 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T15:23:07,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:07,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:07,769 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:07,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-07T15:23:07,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993047775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107087a03b96e7d484490a33a9541fe5024_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730992987108/Put/seqid=0 2024-11-07T15:23:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993047783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993047784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993047786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993047790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T15:23:07,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:07,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:07,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:07,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741945_1121 (size=19474) 2024-11-07T15:23:07,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993047891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993047901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993047902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993047903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:07,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993047903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:07,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T15:23:07,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:07,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:07,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:07,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:07,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993048101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993048111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993048111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993048111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993048111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T15:23:08,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:08,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:08,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:08,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T15:23:08,215 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:08,221 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107087a03b96e7d484490a33a9541fe5024_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107087a03b96e7d484490a33a9541fe5024_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:08,223 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/bfa8a5996d7740fe84526922fc12454f, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:08,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/bfa8a5996d7740fe84526922fc12454f is 175, key is test_row_0/A:col10/1730992987108/Put/seqid=0 2024-11-07T15:23:08,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741946_1122 (size=56733) 2024-11-07T15:23:08,235 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=44, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/bfa8a5996d7740fe84526922fc12454f 2024-11-07T15:23:08,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/d3d78b21fbc44aada807982657450eb4 is 50, key is test_row_0/B:col10/1730992987108/Put/seqid=0 2024-11-07T15:23:08,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741947_1123 (size=12001) 2024-11-07T15:23:08,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/d3d78b21fbc44aada807982657450eb4 2024-11-07T15:23:08,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/3b3cc48af1aa4399a85ddae452202e1e is 50, key is test_row_0/C:col10/1730992987108/Put/seqid=0 2024-11-07T15:23:08,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T15:23:08,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:08,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:08,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:08,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741948_1124 (size=12001) 2024-11-07T15:23:08,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/3b3cc48af1aa4399a85ddae452202e1e 2024-11-07T15:23:08,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/bfa8a5996d7740fe84526922fc12454f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/bfa8a5996d7740fe84526922fc12454f 2024-11-07T15:23:08,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/bfa8a5996d7740fe84526922fc12454f, entries=300, sequenceid=44, filesize=55.4 K 2024-11-07T15:23:08,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/d3d78b21fbc44aada807982657450eb4 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/d3d78b21fbc44aada807982657450eb4 2024-11-07T15:23:08,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/d3d78b21fbc44aada807982657450eb4, entries=150, sequenceid=44, filesize=11.7 K 2024-11-07T15:23:08,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/3b3cc48af1aa4399a85ddae452202e1e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/3b3cc48af1aa4399a85ddae452202e1e 2024-11-07T15:23:08,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/3b3cc48af1aa4399a85ddae452202e1e, entries=150, sequenceid=44, filesize=11.7 K 2024-11-07T15:23:08,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 518e68065b228feb1bbc2642f1a6f79d in 555ms, sequenceid=44, compaction requested=false 2024-11-07T15:23:08,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:08,421 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:08,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:23:08,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T15:23:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:08,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074171f4df299448bab92051038a4f433a_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730992988420/Put/seqid=0 2024-11-07T15:23:08,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741949_1125 (size=14594) 2024-11-07T15:23:08,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993048464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993048468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993048469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993048470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993048471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993048582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993048582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993048582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993048583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993048583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T15:23:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:08,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:08,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T15:23:08,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:08,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:08,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:08,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993048790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993048791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993048791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993048791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993048791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:08,847 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T15:23:08,859 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:08,865 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074171f4df299448bab92051038a4f433a_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074171f4df299448bab92051038a4f433a_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:08,866 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/1538cc2a937045ee918fe8ffb132914b, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:08,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/1538cc2a937045ee918fe8ffb132914b is 175, key is test_row_0/A:col10/1730992988420/Put/seqid=0 2024-11-07T15:23:08,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741950_1126 (size=39549) 2024-11-07T15:23:08,893 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/1538cc2a937045ee918fe8ffb132914b 2024-11-07T15:23:08,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T15:23:08,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:08,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:08,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:08,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:08,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/22ace152cbcd4ea2835b7399b3f7fed8 is 50, key is test_row_0/B:col10/1730992988420/Put/seqid=0 2024-11-07T15:23:08,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741951_1127 (size=12001) 2024-11-07T15:23:09,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T15:23:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:09,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:09,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:09,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993049101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993049101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993049101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993049101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993049101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T15:23:09,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T15:23:09,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:09,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:09,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:09,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:09,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:09,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/22ace152cbcd4ea2835b7399b3f7fed8 2024-11-07T15:23:09,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/8d362e14837842f8ba070fe9c94e50fd is 50, key is test_row_0/C:col10/1730992988420/Put/seqid=0 2024-11-07T15:23:09,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741952_1128 (size=12001) 2024-11-07T15:23:09,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/8d362e14837842f8ba070fe9c94e50fd 2024-11-07T15:23:09,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/1538cc2a937045ee918fe8ffb132914b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/1538cc2a937045ee918fe8ffb132914b 2024-11-07T15:23:09,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T15:23:09,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/1538cc2a937045ee918fe8ffb132914b, entries=200, sequenceid=55, filesize=38.6 K 2024-11-07T15:23:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:09,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:09,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/22ace152cbcd4ea2835b7399b3f7fed8 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/22ace152cbcd4ea2835b7399b3f7fed8 2024-11-07T15:23:09,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/22ace152cbcd4ea2835b7399b3f7fed8, entries=150, sequenceid=55, filesize=11.7 K 2024-11-07T15:23:09,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/8d362e14837842f8ba070fe9c94e50fd as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/8d362e14837842f8ba070fe9c94e50fd 2024-11-07T15:23:09,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/8d362e14837842f8ba070fe9c94e50fd, entries=150, sequenceid=55, filesize=11.7 K 2024-11-07T15:23:09,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 518e68065b228feb1bbc2642f1a6f79d in 959ms, sequenceid=55, compaction requested=true 2024-11-07T15:23:09,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:09,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:09,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:09,380 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:09,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:09,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:09,381 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:09,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:09,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:09,383 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:09,383 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/B is initiating minor compaction (all files) 2024-11-07T15:23:09,383 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/B in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:09,383 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/d9c5a66458974763b6889095e73f882f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/d3d78b21fbc44aada807982657450eb4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/22ace152cbcd4ea2835b7399b3f7fed8] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=35.2 K 2024-11-07T15:23:09,383 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135831 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:09,383 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/A is initiating minor compaction (all files) 2024-11-07T15:23:09,383 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/A in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:09,384 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/133ad1657bce4ee9b05c2eda97eb220e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/bfa8a5996d7740fe84526922fc12454f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/1538cc2a937045ee918fe8ffb132914b] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=132.6 K 2024-11-07T15:23:09,384 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:09,384 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/133ad1657bce4ee9b05c2eda97eb220e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/bfa8a5996d7740fe84526922fc12454f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/1538cc2a937045ee918fe8ffb132914b] 2024-11-07T15:23:09,384 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting d9c5a66458974763b6889095e73f882f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730992987037 2024-11-07T15:23:09,385 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 133ad1657bce4ee9b05c2eda97eb220e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730992987037 2024-11-07T15:23:09,385 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfa8a5996d7740fe84526922fc12454f, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1730992987082 2024-11-07T15:23:09,385 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1538cc2a937045ee918fe8ffb132914b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730992987782 2024-11-07T15:23:09,385 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting d3d78b21fbc44aada807982657450eb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1730992987108 2024-11-07T15:23:09,388 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 22ace152cbcd4ea2835b7399b3f7fed8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730992988411 2024-11-07T15:23:09,402 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:09,412 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#B#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:09,413 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/b1bab0c8a60840598c735df36954bfa2 is 50, key is test_row_0/B:col10/1730992988420/Put/seqid=0 2024-11-07T15:23:09,417 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107655459cec40b40d3b28c9b470fb66ba8_518e68065b228feb1bbc2642f1a6f79d store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:09,421 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107655459cec40b40d3b28c9b470fb66ba8_518e68065b228feb1bbc2642f1a6f79d, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:09,421 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107655459cec40b40d3b28c9b470fb66ba8_518e68065b228feb1bbc2642f1a6f79d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:09,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741953_1129 (size=12104) 2024-11-07T15:23:09,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741954_1130 (size=4469) 2024-11-07T15:23:09,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-07T15:23:09,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:09,513 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T15:23:09,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c2b96571908f41a981df9538d30dc313_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730992988467/Put/seqid=0 2024-11-07T15:23:09,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741955_1131 (size=12154) 2024-11-07T15:23:09,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:09,540 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c2b96571908f41a981df9538d30dc313_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c2b96571908f41a981df9538d30dc313_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:09,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/533d2294abc1442eaa3c01416fa64ac0, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:09,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/533d2294abc1442eaa3c01416fa64ac0 is 175, key is test_row_0/A:col10/1730992988467/Put/seqid=0 2024-11-07T15:23:09,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741956_1132 (size=30955) 2024-11-07T15:23:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:09,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:09,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993049616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993049618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993049618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993049619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993049625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993049731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993049731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993049731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993049740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993049741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,840 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/b1bab0c8a60840598c735df36954bfa2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/b1bab0c8a60840598c735df36954bfa2 2024-11-07T15:23:09,843 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#A#compaction#110 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:09,843 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/480dc28724df40e8abad12be40f75c23 is 175, key is test_row_0/A:col10/1730992988420/Put/seqid=0 2024-11-07T15:23:09,850 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/B of 518e68065b228feb1bbc2642f1a6f79d into b1bab0c8a60840598c735df36954bfa2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:09,850 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:09,850 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/B, priority=13, startTime=1730992989380; duration=0sec 2024-11-07T15:23:09,850 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:09,850 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:B 2024-11-07T15:23:09,850 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:09,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741957_1133 (size=31058) 2024-11-07T15:23:09,853 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:09,853 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/C is initiating minor compaction (all files) 2024-11-07T15:23:09,853 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/C in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:09,853 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/152a520d914a48248da9b02121803000, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/3b3cc48af1aa4399a85ddae452202e1e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/8d362e14837842f8ba070fe9c94e50fd] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=35.2 K 2024-11-07T15:23:09,854 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 152a520d914a48248da9b02121803000, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730992987037 2024-11-07T15:23:09,854 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b3cc48af1aa4399a85ddae452202e1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1730992987108 2024-11-07T15:23:09,854 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d362e14837842f8ba070fe9c94e50fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730992988411 2024-11-07T15:23:09,859 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/480dc28724df40e8abad12be40f75c23 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/480dc28724df40e8abad12be40f75c23 2024-11-07T15:23:09,870 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/A of 518e68065b228feb1bbc2642f1a6f79d into 480dc28724df40e8abad12be40f75c23(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:09,870 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:09,870 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/A, priority=13, startTime=1730992989380; duration=0sec 2024-11-07T15:23:09,870 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:09,870 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:A 2024-11-07T15:23:09,870 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#C#compaction#113 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:09,871 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/be7c80a60bc64b00a4e6b8c44dd1a927 is 50, key is test_row_0/C:col10/1730992988420/Put/seqid=0 2024-11-07T15:23:09,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741958_1134 (size=12104) 2024-11-07T15:23:09,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993049940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993049941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993049941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,949 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/533d2294abc1442eaa3c01416fa64ac0 2024-11-07T15:23:09,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993049951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993049951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:09,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/7658a690d16e43a58ba6f261f7cbbf99 is 50, key is test_row_0/B:col10/1730992988467/Put/seqid=0 2024-11-07T15:23:09,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741959_1135 (size=12001) 2024-11-07T15:23:09,972 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/7658a690d16e43a58ba6f261f7cbbf99 2024-11-07T15:23:09,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/014f7a3e4c494f3497dbf17e8ae899c4 is 50, key is test_row_0/C:col10/1730992988467/Put/seqid=0 2024-11-07T15:23:10,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741960_1136 (size=12001) 2024-11-07T15:23:10,004 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/014f7a3e4c494f3497dbf17e8ae899c4 2024-11-07T15:23:10,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/533d2294abc1442eaa3c01416fa64ac0 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/533d2294abc1442eaa3c01416fa64ac0 2024-11-07T15:23:10,021 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/533d2294abc1442eaa3c01416fa64ac0, entries=150, sequenceid=81, filesize=30.2 K 2024-11-07T15:23:10,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/7658a690d16e43a58ba6f261f7cbbf99 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/7658a690d16e43a58ba6f261f7cbbf99 2024-11-07T15:23:10,038 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/7658a690d16e43a58ba6f261f7cbbf99, entries=150, sequenceid=81, filesize=11.7 K 2024-11-07T15:23:10,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/014f7a3e4c494f3497dbf17e8ae899c4 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/014f7a3e4c494f3497dbf17e8ae899c4 2024-11-07T15:23:10,047 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/014f7a3e4c494f3497dbf17e8ae899c4, entries=150, sequenceid=81, filesize=11.7 K 2024-11-07T15:23:10,049 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 518e68065b228feb1bbc2642f1a6f79d in 536ms, sequenceid=81, compaction requested=false 2024-11-07T15:23:10,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:10,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:10,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-07T15:23:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-07T15:23:10,055 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-07T15:23:10,056 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0250 sec 2024-11-07T15:23:10,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 3.0380 sec 2024-11-07T15:23:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:10,253 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:10,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:23:10,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f043b94e0b0343769f9057e08c36afd7_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730992989617/Put/seqid=0 2024-11-07T15:23:10,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741961_1137 (size=17034) 2024-11-07T15:23:10,275 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,283 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f043b94e0b0343769f9057e08c36afd7_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f043b94e0b0343769f9057e08c36afd7_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:10,285 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/02ee465cc03c43a98260e3205ee2cabe, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:10,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/02ee465cc03c43a98260e3205ee2cabe is 175, key is test_row_0/A:col10/1730992989617/Put/seqid=0 2024-11-07T15:23:10,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993050291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:10,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993050294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:10,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:10,302 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/be7c80a60bc64b00a4e6b8c44dd1a927 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/be7c80a60bc64b00a4e6b8c44dd1a927 2024-11-07T15:23:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993050294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:10,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993050296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:10,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993050297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:10,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741962_1138 (size=48139) 2024-11-07T15:23:10,306 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/02ee465cc03c43a98260e3205ee2cabe 2024-11-07T15:23:10,317 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/C of 518e68065b228feb1bbc2642f1a6f79d into be7c80a60bc64b00a4e6b8c44dd1a927(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:10,317 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:10,318 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/C, priority=13, startTime=1730992989381; duration=0sec 2024-11-07T15:23:10,318 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:10,318 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:C 2024-11-07T15:23:10,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/2ec14b98104e4d1aa06ed619a6359513 is 50, key is test_row_0/B:col10/1730992989617/Put/seqid=0 2024-11-07T15:23:10,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741963_1139 (size=12001) 2024-11-07T15:23:10,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/2ec14b98104e4d1aa06ed619a6359513 2024-11-07T15:23:10,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/39baea90b7d04ee08152b81d7c1abeda is 50, key is test_row_0/C:col10/1730992989617/Put/seqid=0 2024-11-07T15:23:10,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741964_1140 (size=12001) 2024-11-07T15:23:10,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/39baea90b7d04ee08152b81d7c1abeda 2024-11-07T15:23:10,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/02ee465cc03c43a98260e3205ee2cabe as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/02ee465cc03c43a98260e3205ee2cabe 2024-11-07T15:23:10,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/02ee465cc03c43a98260e3205ee2cabe, entries=250, sequenceid=95, filesize=47.0 K 2024-11-07T15:23:10,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/2ec14b98104e4d1aa06ed619a6359513 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/2ec14b98104e4d1aa06ed619a6359513 2024-11-07T15:23:10,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/2ec14b98104e4d1aa06ed619a6359513, entries=150, sequenceid=95, filesize=11.7 K 2024-11-07T15:23:10,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/39baea90b7d04ee08152b81d7c1abeda as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/39baea90b7d04ee08152b81d7c1abeda 2024-11-07T15:23:10,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993050411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:10,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/39baea90b7d04ee08152b81d7c1abeda, entries=150, sequenceid=95, filesize=11.7 K 2024-11-07T15:23:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:10,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 518e68065b228feb1bbc2642f1a6f79d in 165ms, sequenceid=95, compaction requested=true 2024-11-07T15:23:10,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:10,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:10,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:10,419 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:10,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:10,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:10,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:10,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T15:23:10,419 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:10,421 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:10,421 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:10,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T15:23:10,421 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/B is initiating minor compaction (all files) 2024-11-07T15:23:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:10,421 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/B in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:10,421 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/b1bab0c8a60840598c735df36954bfa2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/7658a690d16e43a58ba6f261f7cbbf99, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/2ec14b98104e4d1aa06ed619a6359513] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=35.3 K 2024-11-07T15:23:10,422 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:10,422 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1bab0c8a60840598c735df36954bfa2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730992988411 2024-11-07T15:23:10,422 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/A is initiating minor compaction (all files) 2024-11-07T15:23:10,422 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/A in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:10,422 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/480dc28724df40e8abad12be40f75c23, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/533d2294abc1442eaa3c01416fa64ac0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/02ee465cc03c43a98260e3205ee2cabe] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=107.6 K 2024-11-07T15:23:10,423 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:10,423 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7658a690d16e43a58ba6f261f7cbbf99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1730992988467 2024-11-07T15:23:10,423 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/480dc28724df40e8abad12be40f75c23, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/533d2294abc1442eaa3c01416fa64ac0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/02ee465cc03c43a98260e3205ee2cabe] 2024-11-07T15:23:10,423 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ec14b98104e4d1aa06ed619a6359513, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730992989613 2024-11-07T15:23:10,423 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 480dc28724df40e8abad12be40f75c23, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730992988411 2024-11-07T15:23:10,424 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 533d2294abc1442eaa3c01416fa64ac0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1730992988467 2024-11-07T15:23:10,425 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 02ee465cc03c43a98260e3205ee2cabe, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730992989613 2024-11-07T15:23:10,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993050426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:10,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993050428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:10,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:10,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993050428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:10,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:10,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993050433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:10,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f208c350c35845dcb8334132c0477b94_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730992990420/Put/seqid=0 2024-11-07T15:23:10,458 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#B#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:10,459 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/e5193d78fca6444cb4dddd53a509d108 is 50, key is test_row_0/B:col10/1730992989617/Put/seqid=0 2024-11-07T15:23:10,461 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:10,473 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411073bea325baba6430bb325c756e8dcb10b_518e68065b228feb1bbc2642f1a6f79d store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:10,478 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411073bea325baba6430bb325c756e8dcb10b_518e68065b228feb1bbc2642f1a6f79d, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:10,478 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073bea325baba6430bb325c756e8dcb10b_518e68065b228feb1bbc2642f1a6f79d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:10,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741965_1141 (size=17034) 2024-11-07T15:23:10,480 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,485 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f208c350c35845dcb8334132c0477b94_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f208c350c35845dcb8334132c0477b94_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:10,486 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/d18becaefd344c8b9cc388cdce6fc082, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:10,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/d18becaefd344c8b9cc388cdce6fc082 is 175, key is test_row_0/A:col10/1730992990420/Put/seqid=0 2024-11-07T15:23:10,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741966_1142 (size=12207) 2024-11-07T15:23:10,506 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/e5193d78fca6444cb4dddd53a509d108 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/e5193d78fca6444cb4dddd53a509d108 2024-11-07T15:23:10,515 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/B of 518e68065b228feb1bbc2642f1a6f79d into e5193d78fca6444cb4dddd53a509d108(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:10,515 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:10,515 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/B, priority=13, startTime=1730992990419; duration=0sec 2024-11-07T15:23:10,515 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:10,515 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:B 2024-11-07T15:23:10,515 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:10,517 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:10,517 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/C is initiating minor compaction (all files) 2024-11-07T15:23:10,517 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/C in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:10,517 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/be7c80a60bc64b00a4e6b8c44dd1a927, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/014f7a3e4c494f3497dbf17e8ae899c4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/39baea90b7d04ee08152b81d7c1abeda] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=35.3 K 2024-11-07T15:23:10,517 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting be7c80a60bc64b00a4e6b8c44dd1a927, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730992988411 2024-11-07T15:23:10,518 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 014f7a3e4c494f3497dbf17e8ae899c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1730992988467 2024-11-07T15:23:10,518 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39baea90b7d04ee08152b81d7c1abeda, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730992989613 2024-11-07T15:23:10,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741968_1144 (size=48139) 2024-11-07T15:23:10,528 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=124, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/d18becaefd344c8b9cc388cdce6fc082 2024-11-07T15:23:10,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741967_1143 (size=4469) 2024-11-07T15:23:10,532 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#A#compaction#121 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:10,533 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/36377a37cf89434fa0b8c33274e8d0de is 175, key is test_row_0/A:col10/1730992989617/Put/seqid=0 2024-11-07T15:23:10,543 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#C#compaction#122 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:10,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993050541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:10,544 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/7a5d707c9eda4e54a8d0712a679f1a31 is 50, key is test_row_0/C:col10/1730992989617/Put/seqid=0 2024-11-07T15:23:10,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:10,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993050541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:10,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:10,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993050541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:10,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:10,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993050544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:10,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:10,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/52cf0ab26ec54cd5ad3e405ba7658058 is 50, key is test_row_0/B:col10/1730992990420/Put/seqid=0 2024-11-07T15:23:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:10,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741969_1145 (size=31161) 2024-11-07T15:23:10,569 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/36377a37cf89434fa0b8c33274e8d0de as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/36377a37cf89434fa0b8c33274e8d0de 2024-11-07T15:23:10,580 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/A of 518e68065b228feb1bbc2642f1a6f79d into 36377a37cf89434fa0b8c33274e8d0de(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:10,580 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:10,580 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/A, priority=13, startTime=1730992990418; duration=0sec 2024-11-07T15:23:10,580 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:10,580 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:A 2024-11-07T15:23:10,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741970_1146 (size=12001) 2024-11-07T15:23:10,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/52cf0ab26ec54cd5ad3e405ba7658058 2024-11-07T15:23:10,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741971_1147 (size=12207) 2024-11-07T15:23:10,600 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/7a5d707c9eda4e54a8d0712a679f1a31 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/7a5d707c9eda4e54a8d0712a679f1a31 2024-11-07T15:23:10,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/33a5449892164896b60db5c28b53052c is 50, key is test_row_0/C:col10/1730992990420/Put/seqid=0 2024-11-07T15:23:10,611 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/C of 518e68065b228feb1bbc2642f1a6f79d into 7a5d707c9eda4e54a8d0712a679f1a31(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:10,611 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:10,611 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/C, priority=13, startTime=1730992990419; duration=0sec 2024-11-07T15:23:10,611 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:10,611 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:C 2024-11-07T15:23:10,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741972_1148 (size=12001) 2024-11-07T15:23:10,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/33a5449892164896b60db5c28b53052c 2024-11-07T15:23:10,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993050620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:10,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/d18becaefd344c8b9cc388cdce6fc082 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/d18becaefd344c8b9cc388cdce6fc082 2024-11-07T15:23:10,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/d18becaefd344c8b9cc388cdce6fc082, entries=250, sequenceid=124, filesize=47.0 K 2024-11-07T15:23:10,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/52cf0ab26ec54cd5ad3e405ba7658058 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/52cf0ab26ec54cd5ad3e405ba7658058 2024-11-07T15:23:10,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/52cf0ab26ec54cd5ad3e405ba7658058, entries=150, sequenceid=124, filesize=11.7 K 2024-11-07T15:23:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/33a5449892164896b60db5c28b53052c as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/33a5449892164896b60db5c28b53052c 2024-11-07T15:23:10,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/33a5449892164896b60db5c28b53052c, entries=150, sequenceid=124, filesize=11.7 K 2024-11-07T15:23:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 518e68065b228feb1bbc2642f1a6f79d in 245ms, sequenceid=124, compaction requested=false 2024-11-07T15:23:10,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:10,775 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:10,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:23:10,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cb620feccdb842709f988038de1e868a_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730992990770/Put/seqid=0 2024-11-07T15:23:10,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741973_1149 (size=14794) 2024-11-07T15:23:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:10,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:11,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993050973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993050976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993051002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993051005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993051007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993051111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993051111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993051130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993051130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993051130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-07T15:23:11,202 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:11,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-07T15:23:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T15:23:11,206 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:11,207 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:11,208 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:11,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:11,214 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cb620feccdb842709f988038de1e868a_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cb620feccdb842709f988038de1e868a_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:11,215 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/ff18382923c945509fd089c4dd539b70, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:11,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/ff18382923c945509fd089c4dd539b70 is 175, key is test_row_0/A:col10/1730992990770/Put/seqid=0 2024-11-07T15:23:11,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741974_1150 (size=39745) 2024-11-07T15:23:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T15:23:11,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993051320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993051320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993051341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993051341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993051341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T15:23:11,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:11,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:11,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T15:23:11,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:11,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:11,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:11,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:11,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T15:23:11,621 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/ff18382923c945509fd089c4dd539b70 2024-11-07T15:23:11,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/89c9b34e63104a1eaf253e4dce3663df is 50, key is test_row_0/B:col10/1730992990770/Put/seqid=0 2024-11-07T15:23:11,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993051631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993051631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741975_1151 (size=9757) 2024-11-07T15:23:11,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/89c9b34e63104a1eaf253e4dce3663df 2024-11-07T15:23:11,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993051652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993051652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993051652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:11,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:11,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/532965b432a3457396b7bcc75549f894 is 50, key is test_row_0/C:col10/1730992990770/Put/seqid=0 2024-11-07T15:23:11,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T15:23:11,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:11,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:11,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:11,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:11,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:11,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:11,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741976_1152 (size=9757) 2024-11-07T15:23:11,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/532965b432a3457396b7bcc75549f894 2024-11-07T15:23:11,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/ff18382923c945509fd089c4dd539b70 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ff18382923c945509fd089c4dd539b70 2024-11-07T15:23:11,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ff18382923c945509fd089c4dd539b70, entries=200, sequenceid=139, filesize=38.8 K 2024-11-07T15:23:11,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/89c9b34e63104a1eaf253e4dce3663df as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/89c9b34e63104a1eaf253e4dce3663df 2024-11-07T15:23:11,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/89c9b34e63104a1eaf253e4dce3663df, entries=100, sequenceid=139, filesize=9.5 K 2024-11-07T15:23:11,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/532965b432a3457396b7bcc75549f894 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/532965b432a3457396b7bcc75549f894 2024-11-07T15:23:11,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/532965b432a3457396b7bcc75549f894, entries=100, sequenceid=139, filesize=9.5 K 2024-11-07T15:23:11,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 518e68065b228feb1bbc2642f1a6f79d in 956ms, sequenceid=139, compaction requested=true 2024-11-07T15:23:11,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:11,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:11,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:11,731 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:11,732 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:11,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:11,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:11,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:11,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:11,733 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119045 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:11,733 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/A is initiating minor compaction (all files) 2024-11-07T15:23:11,733 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/A in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:11,733 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/36377a37cf89434fa0b8c33274e8d0de, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/d18becaefd344c8b9cc388cdce6fc082, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ff18382923c945509fd089c4dd539b70] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=116.3 K 2024-11-07T15:23:11,733 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:11,734 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/36377a37cf89434fa0b8c33274e8d0de, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/d18becaefd344c8b9cc388cdce6fc082, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ff18382923c945509fd089c4dd539b70] 2024-11-07T15:23:11,734 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:11,734 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/B is initiating minor compaction (all files) 2024-11-07T15:23:11,734 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/B in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:11,734 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/e5193d78fca6444cb4dddd53a509d108, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/52cf0ab26ec54cd5ad3e405ba7658058, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/89c9b34e63104a1eaf253e4dce3663df] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=33.2 K 2024-11-07T15:23:11,735 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting e5193d78fca6444cb4dddd53a509d108, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730992989613 2024-11-07T15:23:11,735 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36377a37cf89434fa0b8c33274e8d0de, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730992989613 2024-11-07T15:23:11,735 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 52cf0ab26ec54cd5ad3e405ba7658058, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1730992990290 2024-11-07T15:23:11,735 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting d18becaefd344c8b9cc388cdce6fc082, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1730992990290 2024-11-07T15:23:11,736 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 89c9b34e63104a1eaf253e4dce3663df, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1730992990768 2024-11-07T15:23:11,736 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff18382923c945509fd089c4dd539b70, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1730992990754 2024-11-07T15:23:11,749 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:11,753 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#B#compaction#129 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:11,754 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/85f40ba022a747879e8ac394c83e86d6 is 50, key is test_row_0/B:col10/1730992990770/Put/seqid=0 2024-11-07T15:23:11,761 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107b85973e775b7426eac5b15f23bb1504c_518e68065b228feb1bbc2642f1a6f79d store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:11,763 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107b85973e775b7426eac5b15f23bb1504c_518e68065b228feb1bbc2642f1a6f79d, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:11,763 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b85973e775b7426eac5b15f23bb1504c_518e68065b228feb1bbc2642f1a6f79d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:11,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741977_1153 (size=12409) 2024-11-07T15:23:11,795 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/85f40ba022a747879e8ac394c83e86d6 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/85f40ba022a747879e8ac394c83e86d6 2024-11-07T15:23:11,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741978_1154 (size=4469) 2024-11-07T15:23:11,802 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#A#compaction#128 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:11,803 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/d7a721decf6f49ef86947b374092cfd6 is 175, key is test_row_0/A:col10/1730992990770/Put/seqid=0 2024-11-07T15:23:11,806 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/B of 518e68065b228feb1bbc2642f1a6f79d into 85f40ba022a747879e8ac394c83e86d6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:11,806 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:11,806 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/B, priority=13, startTime=1730992991732; duration=0sec 2024-11-07T15:23:11,806 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:11,806 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:B 2024-11-07T15:23:11,806 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:11,808 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:11,808 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/C is initiating minor compaction (all files) 2024-11-07T15:23:11,808 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/C in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:11,808 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/7a5d707c9eda4e54a8d0712a679f1a31, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/33a5449892164896b60db5c28b53052c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/532965b432a3457396b7bcc75549f894] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=33.2 K 2024-11-07T15:23:11,809 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a5d707c9eda4e54a8d0712a679f1a31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730992989613 2024-11-07T15:23:11,810 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 33a5449892164896b60db5c28b53052c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1730992990290 2024-11-07T15:23:11,812 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 532965b432a3457396b7bcc75549f894, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1730992990768 2024-11-07T15:23:11,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741979_1155 (size=31470) 2024-11-07T15:23:11,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T15:23:11,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:11,827 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T15:23:11,830 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#C#compaction#130 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T15:23:11,831 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/5749a265dc1941388134236f2666a8cb is 50, key is test_row_0/C:col10/1730992990770/Put/seqid=0 2024-11-07T15:23:11,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b4147144ded6449c99b0e3f7e9d72c10_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730992990998/Put/seqid=0 2024-11-07T15:23:11,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741980_1156 (size=12409) 2024-11-07T15:23:11,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741981_1157 (size=12304) 2024-11-07T15:23:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:12,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:12,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993052160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:12,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:12,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993052160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:12,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993052160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:12,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993052161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:12,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993052162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:12,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993052270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:12,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993052270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:12,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:12,313 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b4147144ded6449c99b0e3f7e9d72c10_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b4147144ded6449c99b0e3f7e9d72c10_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:12,314 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/d7a721decf6f49ef86947b374092cfd6 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/d7a721decf6f49ef86947b374092cfd6 2024-11-07T15:23:12,315 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/5749a265dc1941388134236f2666a8cb as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/5749a265dc1941388134236f2666a8cb 2024-11-07T15:23:12,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/f0e872621c544addb0612913a72b4022, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:12,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/f0e872621c544addb0612913a72b4022 is 175, key is test_row_0/A:col10/1730992990998/Put/seqid=0 2024-11-07T15:23:12,321 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/A of 518e68065b228feb1bbc2642f1a6f79d into d7a721decf6f49ef86947b374092cfd6(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:12,321 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:12,321 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/A, priority=13, startTime=1730992991731; duration=0sec 2024-11-07T15:23:12,322 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:12,322 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:A 2024-11-07T15:23:12,323 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/C of 518e68065b228feb1bbc2642f1a6f79d into 5749a265dc1941388134236f2666a8cb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:12,323 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:12,323 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/C, priority=13, startTime=1730992991733; duration=0sec 2024-11-07T15:23:12,323 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:12,323 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:C 2024-11-07T15:23:12,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741982_1158 (size=31105) 2024-11-07T15:23:12,326 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/f0e872621c544addb0612913a72b4022 2024-11-07T15:23:12,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/725e03e4697d40c899b082b40da7eab2 is 50, key is test_row_0/B:col10/1730992990998/Put/seqid=0 2024-11-07T15:23:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T15:23:12,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741983_1159 (size=12151) 2024-11-07T15:23:12,350 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/725e03e4697d40c899b082b40da7eab2 2024-11-07T15:23:12,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/20e0b5a58774472d947b005599285432 is 50, key is test_row_0/C:col10/1730992990998/Put/seqid=0 2024-11-07T15:23:12,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741984_1160 (size=12151) 2024-11-07T15:23:12,388 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/20e0b5a58774472d947b005599285432 2024-11-07T15:23:12,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/f0e872621c544addb0612913a72b4022 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/f0e872621c544addb0612913a72b4022 2024-11-07T15:23:12,406 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/f0e872621c544addb0612913a72b4022, entries=150, sequenceid=165, filesize=30.4 K 2024-11-07T15:23:12,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/725e03e4697d40c899b082b40da7eab2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/725e03e4697d40c899b082b40da7eab2 2024-11-07T15:23:12,413 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/725e03e4697d40c899b082b40da7eab2, entries=150, sequenceid=165, filesize=11.9 K 2024-11-07T15:23:12,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/20e0b5a58774472d947b005599285432 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/20e0b5a58774472d947b005599285432 2024-11-07T15:23:12,422 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/20e0b5a58774472d947b005599285432, entries=150, sequenceid=165, filesize=11.9 K 2024-11-07T15:23:12,423 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 518e68065b228feb1bbc2642f1a6f79d in 596ms, sequenceid=165, compaction requested=false 2024-11-07T15:23:12,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:12,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:12,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-07T15:23:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-07T15:23:12,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-07T15:23:12,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2170 sec 2024-11-07T15:23:12,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.2240 sec 2024-11-07T15:23:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:12,482 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:12,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T15:23:12,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072ce2b9431dea4a43a125afba0cc41133_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730992992155/Put/seqid=0 2024-11-07T15:23:12,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741985_1161 (size=14794) 2024-11-07T15:23:12,512 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:12,520 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072ce2b9431dea4a43a125afba0cc41133_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072ce2b9431dea4a43a125afba0cc41133_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:12,522 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/b153c66000ff45ae892c92ef7c941a04, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:12,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/b153c66000ff45ae892c92ef7c941a04 is 175, key is test_row_0/A:col10/1730992992155/Put/seqid=0 2024-11-07T15:23:12,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741986_1162 (size=39749) 2024-11-07T15:23:12,539 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/b153c66000ff45ae892c92ef7c941a04 2024-11-07T15:23:12,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/552fa2e954b44e49b69d9fe08797554d is 50, key is test_row_0/B:col10/1730992992155/Put/seqid=0 2024-11-07T15:23:12,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993052549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:12,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:12,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993052554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:12,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741987_1163 (size=12151) 2024-11-07T15:23:12,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/552fa2e954b44e49b69d9fe08797554d 2024-11-07T15:23:12,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/ef1c8b75d1d34e81ba0a39c618dc4268 is 50, key is test_row_0/C:col10/1730992992155/Put/seqid=0 2024-11-07T15:23:12,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741988_1164 (size=12151) 2024-11-07T15:23:12,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993052661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:12,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:12,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993052670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:12,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:12,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993052871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:12,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:12,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:12,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993052880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:12,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:12,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:12,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/ef1c8b75d1d34e81ba0a39c618dc4268 2024-11-07T15:23:12,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/b153c66000ff45ae892c92ef7c941a04 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/b153c66000ff45ae892c92ef7c941a04 2024-11-07T15:23:13,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/b153c66000ff45ae892c92ef7c941a04, entries=200, sequenceid=179, filesize=38.8 K 2024-11-07T15:23:13,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/552fa2e954b44e49b69d9fe08797554d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/552fa2e954b44e49b69d9fe08797554d 2024-11-07T15:23:13,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/552fa2e954b44e49b69d9fe08797554d, entries=150, sequenceid=179, filesize=11.9 K 2024-11-07T15:23:13,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/ef1c8b75d1d34e81ba0a39c618dc4268 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/ef1c8b75d1d34e81ba0a39c618dc4268 2024-11-07T15:23:13,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/ef1c8b75d1d34e81ba0a39c618dc4268, entries=150, sequenceid=179, filesize=11.9 K 2024-11-07T15:23:13,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 518e68065b228feb1bbc2642f1a6f79d in 535ms, sequenceid=179, compaction requested=true 2024-11-07T15:23:13,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:13,017 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:13,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:13,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:13,018 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:13,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:13,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:13,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:13,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:13,019 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102324 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:13,019 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/A is initiating minor compaction (all files) 2024-11-07T15:23:13,019 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/A in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:13,019 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:13,019 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/B is initiating minor compaction (all files) 2024-11-07T15:23:13,019 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/d7a721decf6f49ef86947b374092cfd6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/f0e872621c544addb0612913a72b4022, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/b153c66000ff45ae892c92ef7c941a04] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=99.9 K 2024-11-07T15:23:13,019 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/B in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:13,019 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:13,019 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/d7a721decf6f49ef86947b374092cfd6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/f0e872621c544addb0612913a72b4022, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/b153c66000ff45ae892c92ef7c941a04] 2024-11-07T15:23:13,019 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/85f40ba022a747879e8ac394c83e86d6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/725e03e4697d40c899b082b40da7eab2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/552fa2e954b44e49b69d9fe08797554d] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=35.9 K 2024-11-07T15:23:13,020 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7a721decf6f49ef86947b374092cfd6, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1730992990290 2024-11-07T15:23:13,020 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 85f40ba022a747879e8ac394c83e86d6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1730992990290 2024-11-07T15:23:13,020 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 725e03e4697d40c899b082b40da7eab2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1730992990967 2024-11-07T15:23:13,020 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0e872621c544addb0612913a72b4022, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1730992990967 2024-11-07T15:23:13,021 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 552fa2e954b44e49b69d9fe08797554d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1730992992155 2024-11-07T15:23:13,022 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting b153c66000ff45ae892c92ef7c941a04, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1730992992155 2024-11-07T15:23:13,037 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:13,038 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#B#compaction#137 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:13,039 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/18eda101d355472ca413ba971e4e27ae is 50, key is test_row_0/B:col10/1730992992155/Put/seqid=0 2024-11-07T15:23:13,041 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411076b5986f12f53436fba3d30f0e890b659_518e68065b228feb1bbc2642f1a6f79d store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:13,043 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411076b5986f12f53436fba3d30f0e890b659_518e68065b228feb1bbc2642f1a6f79d, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:13,043 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076b5986f12f53436fba3d30f0e890b659_518e68065b228feb1bbc2642f1a6f79d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:13,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741989_1165 (size=4469) 2024-11-07T15:23:13,064 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#A#compaction#138 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:13,064 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/5ea3e99eef004c8b8addd1d2376a49f8 is 175, key is test_row_0/A:col10/1730992992155/Put/seqid=0 2024-11-07T15:23:13,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741990_1166 (size=12561) 2024-11-07T15:23:13,077 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/18eda101d355472ca413ba971e4e27ae as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/18eda101d355472ca413ba971e4e27ae 2024-11-07T15:23:13,086 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/B of 518e68065b228feb1bbc2642f1a6f79d into 18eda101d355472ca413ba971e4e27ae(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:13,086 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:13,086 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/B, priority=13, startTime=1730992993018; duration=0sec 2024-11-07T15:23:13,087 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:13,087 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:B 2024-11-07T15:23:13,088 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:13,090 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:13,090 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/C is initiating minor compaction (all files) 2024-11-07T15:23:13,090 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/C in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:13,091 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/5749a265dc1941388134236f2666a8cb, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/20e0b5a58774472d947b005599285432, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/ef1c8b75d1d34e81ba0a39c618dc4268] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=35.9 K 2024-11-07T15:23:13,091 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 5749a265dc1941388134236f2666a8cb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1730992990290 2024-11-07T15:23:13,094 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 20e0b5a58774472d947b005599285432, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1730992990967 2024-11-07T15:23:13,095 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting ef1c8b75d1d34e81ba0a39c618dc4268, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1730992992155 2024-11-07T15:23:13,113 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#C#compaction#139 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:13,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741991_1167 (size=31515) 2024-11-07T15:23:13,115 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/0e3400c9047b4f5b98c8bfeead29845d is 50, key is test_row_0/C:col10/1730992992155/Put/seqid=0 2024-11-07T15:23:13,123 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/5ea3e99eef004c8b8addd1d2376a49f8 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/5ea3e99eef004c8b8addd1d2376a49f8 2024-11-07T15:23:13,130 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/A of 518e68065b228feb1bbc2642f1a6f79d into 5ea3e99eef004c8b8addd1d2376a49f8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:13,130 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:13,130 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/A, priority=13, startTime=1730992993017; duration=0sec 2024-11-07T15:23:13,132 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:13,132 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:A 2024-11-07T15:23:13,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741992_1168 (size=12561) 2024-11-07T15:23:13,155 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/0e3400c9047b4f5b98c8bfeead29845d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/0e3400c9047b4f5b98c8bfeead29845d 2024-11-07T15:23:13,163 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/C of 518e68065b228feb1bbc2642f1a6f79d into 0e3400c9047b4f5b98c8bfeead29845d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:13,163 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:13,163 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/C, priority=13, startTime=1730992993018; duration=0sec 2024-11-07T15:23:13,163 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:13,163 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:C 2024-11-07T15:23:13,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:13,173 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:13,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T15:23:13,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993053189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077b98b682e5ca4a24907e4b019b9a5c1b_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730992992538/Put/seqid=0 2024-11-07T15:23:13,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993053190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993053191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993053191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993053192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741993_1169 (size=14794) 2024-11-07T15:23:13,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993053301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993053301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993053301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993053301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T15:23:13,351 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:13,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-07T15:23:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T15:23:13,354 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:13,354 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:13,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T15:23:13,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T15:23:13,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:13,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:13,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:13,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:13,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:13,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993053510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993053511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993053511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993053511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,614 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:13,619 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077b98b682e5ca4a24907e4b019b9a5c1b_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077b98b682e5ca4a24907e4b019b9a5c1b_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:13,619 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/bf0d0d082a334e06865022b1f3b55f5a, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:13,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/bf0d0d082a334e06865022b1f3b55f5a is 175, key is test_row_0/A:col10/1730992992538/Put/seqid=0 2024-11-07T15:23:13,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741994_1170 (size=39749) 2024-11-07T15:23:13,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T15:23:13,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:13,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:13,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:13,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:13,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T15:23:13,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993053701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T15:23:13,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:13,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:13,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:13,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:13,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:13,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:13,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993053821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993053821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993053821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:13,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993053821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:13,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:13,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T15:23:13,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:13,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:13,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:13,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:13,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T15:23:14,025 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/bf0d0d082a334e06865022b1f3b55f5a 2024-11-07T15:23:14,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/6a48dd5e6b0a43218ee9238afddc40b0 is 50, key is test_row_0/B:col10/1730992992538/Put/seqid=0 2024-11-07T15:23:14,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741995_1171 (size=12151) 2024-11-07T15:23:14,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T15:23:14,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:14,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:14,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:14,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:14,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:14,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T15:23:14,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:14,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:14,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:14,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:14,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:14,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993054331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:14,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:14,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:14,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993054331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993054331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:14,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993054331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:14,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T15:23:14,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:14,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:14,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:14,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:14,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:14,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/6a48dd5e6b0a43218ee9238afddc40b0 2024-11-07T15:23:14,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/4a4c5b964e6e4d1586ec05e3010f9905 is 50, key is test_row_0/C:col10/1730992992538/Put/seqid=0 2024-11-07T15:23:14,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741996_1172 (size=12151) 2024-11-07T15:23:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T15:23:14,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T15:23:14,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:14,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:14,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:14,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:14,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:14,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993054711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:14,716 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_0' in 518e68065b228feb1bbc2642f1a6f79d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:23:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:14,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T15:23:14,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:14,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:14,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:14,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:14,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:14,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/4a4c5b964e6e4d1586ec05e3010f9905 2024-11-07T15:23:14,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/bf0d0d082a334e06865022b1f3b55f5a as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/bf0d0d082a334e06865022b1f3b55f5a 2024-11-07T15:23:14,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/bf0d0d082a334e06865022b1f3b55f5a, entries=200, sequenceid=207, filesize=38.8 K 2024-11-07T15:23:14,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/6a48dd5e6b0a43218ee9238afddc40b0 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/6a48dd5e6b0a43218ee9238afddc40b0 2024-11-07T15:23:14,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/6a48dd5e6b0a43218ee9238afddc40b0, entries=150, sequenceid=207, filesize=11.9 K 2024-11-07T15:23:14,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/4a4c5b964e6e4d1586ec05e3010f9905 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/4a4c5b964e6e4d1586ec05e3010f9905 2024-11-07T15:23:14,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/4a4c5b964e6e4d1586ec05e3010f9905, entries=150, sequenceid=207, filesize=11.9 K 2024-11-07T15:23:14,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 518e68065b228feb1bbc2642f1a6f79d in 1709ms, sequenceid=207, compaction requested=false 2024-11-07T15:23:14,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:14,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T15:23:14,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:14,884 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:23:14,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076bb5d25879a746ccaeec5d871e706bfc_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730992993190/Put/seqid=0 2024-11-07T15:23:14,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741997_1173 (size=12304) 2024-11-07T15:23:15,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:15,311 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076bb5d25879a746ccaeec5d871e706bfc_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076bb5d25879a746ccaeec5d871e706bfc_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:15,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/a66a4e4e9c6b4d2eb872a85e75ce6c80, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:15,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/a66a4e4e9c6b4d2eb872a85e75ce6c80 is 175, key is test_row_0/A:col10/1730992993190/Put/seqid=0 2024-11-07T15:23:15,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741998_1174 (size=31105) 2024-11-07T15:23:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:15,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:15,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993055371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:15,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993055372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:15,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:15,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993055373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:15,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:15,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993055373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:15,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993055481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:15,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:15,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993055481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:15,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993055481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993055481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:15,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T15:23:15,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993055690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993055690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993055690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993055690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:15,732 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/a66a4e4e9c6b4d2eb872a85e75ce6c80 2024-11-07T15:23:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/6f201245cf934af0b11bd6ffe11c18cd is 50, key is test_row_0/B:col10/1730992993190/Put/seqid=0 2024-11-07T15:23:15,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741999_1175 (size=12151) 2024-11-07T15:23:15,744 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/6f201245cf934af0b11bd6ffe11c18cd 2024-11-07T15:23:15,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/053995d4d8a24d86b70936a56b06e943 is 50, key is test_row_0/C:col10/1730992993190/Put/seqid=0 2024-11-07T15:23:15,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742000_1176 (size=12151) 2024-11-07T15:23:16,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993056001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993056001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993056001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993056001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,158 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/053995d4d8a24d86b70936a56b06e943 2024-11-07T15:23:16,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/a66a4e4e9c6b4d2eb872a85e75ce6c80 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/a66a4e4e9c6b4d2eb872a85e75ce6c80 2024-11-07T15:23:16,169 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/a66a4e4e9c6b4d2eb872a85e75ce6c80, entries=150, sequenceid=218, filesize=30.4 K 2024-11-07T15:23:16,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/6f201245cf934af0b11bd6ffe11c18cd as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/6f201245cf934af0b11bd6ffe11c18cd 2024-11-07T15:23:16,175 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/6f201245cf934af0b11bd6ffe11c18cd, entries=150, sequenceid=218, filesize=11.9 K 2024-11-07T15:23:16,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/053995d4d8a24d86b70936a56b06e943 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/053995d4d8a24d86b70936a56b06e943 2024-11-07T15:23:16,180 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/053995d4d8a24d86b70936a56b06e943, entries=150, sequenceid=218, filesize=11.9 K 2024-11-07T15:23:16,182 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 518e68065b228feb1bbc2642f1a6f79d in 1297ms, sequenceid=218, compaction requested=true 2024-11-07T15:23:16,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:16,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:16,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-07T15:23:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-07T15:23:16,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-07T15:23:16,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8280 sec 2024-11-07T15:23:16,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.8330 sec 2024-11-07T15:23:16,349 INFO [master/7b3e4e8e12ef:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-07T15:23:16,349 INFO [master/7b3e4e8e12ef:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-07T15:23:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:16,512 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:16,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-07T15:23:16,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072dcef0dd0d2044acb5c0540686b23f7a_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730992996511/Put/seqid=0 2024-11-07T15:23:16,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742001_1177 (size=22268) 2024-11-07T15:23:16,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993056521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993056565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993056566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993056566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993056671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993056681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993056681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993056681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993056731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,732 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_0' in 518e68065b228feb1bbc2642f1a6f79d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:23:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993056881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993056890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993056890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993056890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:16,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:16,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:16,931 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:16,936 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072dcef0dd0d2044acb5c0540686b23f7a_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072dcef0dd0d2044acb5c0540686b23f7a_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:16,937 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/9975d3d0741d4cd3873e5b45c1e78747, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:16,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/9975d3d0741d4cd3873e5b45c1e78747 is 175, key is test_row_0/A:col10/1730992996511/Put/seqid=0 2024-11-07T15:23:16,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742002_1178 (size=65673) 2024-11-07T15:23:16,942 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/9975d3d0741d4cd3873e5b45c1e78747 2024-11-07T15:23:16,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/5ad32bbf6cff4bec8d7a04c6569a14cf is 50, key is test_row_0/B:col10/1730992996511/Put/seqid=0 2024-11-07T15:23:16,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742003_1179 (size=12151) 2024-11-07T15:23:16,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/5ad32bbf6cff4bec8d7a04c6569a14cf 2024-11-07T15:23:16,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/f2c522d6d1e34dd99ea9631b6a3a871d is 50, key is test_row_0/C:col10/1730992996511/Put/seqid=0 2024-11-07T15:23:16,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742004_1180 (size=12151) 2024-11-07T15:23:17,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993057191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:17,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993057201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:17,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993057201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:17,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993057201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:17,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:17,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/f2c522d6d1e34dd99ea9631b6a3a871d 2024-11-07T15:23:17,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/9975d3d0741d4cd3873e5b45c1e78747 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/9975d3d0741d4cd3873e5b45c1e78747 2024-11-07T15:23:17,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/9975d3d0741d4cd3873e5b45c1e78747, entries=350, sequenceid=248, filesize=64.1 K 2024-11-07T15:23:17,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/5ad32bbf6cff4bec8d7a04c6569a14cf as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/5ad32bbf6cff4bec8d7a04c6569a14cf 2024-11-07T15:23:17,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/5ad32bbf6cff4bec8d7a04c6569a14cf, entries=150, sequenceid=248, filesize=11.9 K 2024-11-07T15:23:17,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/f2c522d6d1e34dd99ea9631b6a3a871d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/f2c522d6d1e34dd99ea9631b6a3a871d 2024-11-07T15:23:17,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/f2c522d6d1e34dd99ea9631b6a3a871d, entries=150, sequenceid=248, filesize=11.9 K 2024-11-07T15:23:17,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 518e68065b228feb1bbc2642f1a6f79d in 887ms, sequenceid=248, compaction requested=true 2024-11-07T15:23:17,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:17,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:17,400 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:23:17,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:17,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:17,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:17,400 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:23:17,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:17,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:17,402 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 168042 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:23:17,402 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/A is initiating minor compaction (all files) 2024-11-07T15:23:17,402 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/A in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:17,402 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/5ea3e99eef004c8b8addd1d2376a49f8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/bf0d0d082a334e06865022b1f3b55f5a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/a66a4e4e9c6b4d2eb872a85e75ce6c80, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/9975d3d0741d4cd3873e5b45c1e78747] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=164.1 K 2024-11-07T15:23:17,402 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:17,402 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/5ea3e99eef004c8b8addd1d2376a49f8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/bf0d0d082a334e06865022b1f3b55f5a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/a66a4e4e9c6b4d2eb872a85e75ce6c80, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/9975d3d0741d4cd3873e5b45c1e78747] 2024-11-07T15:23:17,403 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:23:17,403 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/B is initiating minor compaction (all files) 2024-11-07T15:23:17,403 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/B in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:17,403 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/18eda101d355472ca413ba971e4e27ae, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/6a48dd5e6b0a43218ee9238afddc40b0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/6f201245cf934af0b11bd6ffe11c18cd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/5ad32bbf6cff4bec8d7a04c6569a14cf] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=47.9 K 2024-11-07T15:23:17,403 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ea3e99eef004c8b8addd1d2376a49f8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1730992992155 2024-11-07T15:23:17,404 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 18eda101d355472ca413ba971e4e27ae, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1730992992155 2024-11-07T15:23:17,404 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf0d0d082a334e06865022b1f3b55f5a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1730992992531 2024-11-07T15:23:17,404 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting a66a4e4e9c6b4d2eb872a85e75ce6c80, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1730992993186 2024-11-07T15:23:17,404 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a48dd5e6b0a43218ee9238afddc40b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1730992992538 2024-11-07T15:23:17,405 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9975d3d0741d4cd3873e5b45c1e78747, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730992995370 2024-11-07T15:23:17,405 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f201245cf934af0b11bd6ffe11c18cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1730992993186 2024-11-07T15:23:17,405 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ad32bbf6cff4bec8d7a04c6569a14cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730992995372 2024-11-07T15:23:17,415 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#B#compaction#149 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:17,415 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/53ec97f1e3484c88ba735d6ecdcdb0a2 is 50, key is test_row_0/B:col10/1730992996511/Put/seqid=0 2024-11-07T15:23:17,416 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:17,425 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110727485205016c4b25ade9707ed1e7212d_518e68065b228feb1bbc2642f1a6f79d store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:17,427 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110727485205016c4b25ade9707ed1e7212d_518e68065b228feb1bbc2642f1a6f79d, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:17,427 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110727485205016c4b25ade9707ed1e7212d_518e68065b228feb1bbc2642f1a6f79d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:17,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742005_1181 (size=12697) 2024-11-07T15:23:17,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742006_1182 (size=4469) 2024-11-07T15:23:17,447 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#A#compaction#150 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:17,447 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/f1af365f5c8d4536a8563d9cee00b579 is 175, key is test_row_0/A:col10/1730992996511/Put/seqid=0 2024-11-07T15:23:17,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742007_1183 (size=31651) 2024-11-07T15:23:17,466 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/f1af365f5c8d4536a8563d9cee00b579 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/f1af365f5c8d4536a8563d9cee00b579 2024-11-07T15:23:17,472 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/A of 518e68065b228feb1bbc2642f1a6f79d into f1af365f5c8d4536a8563d9cee00b579(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:17,472 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:17,472 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/A, priority=12, startTime=1730992997400; duration=0sec 2024-11-07T15:23:17,472 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:17,472 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:A 2024-11-07T15:23:17,472 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:23:17,474 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:23:17,474 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/C is initiating minor compaction (all files) 2024-11-07T15:23:17,474 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/C in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:17,474 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/0e3400c9047b4f5b98c8bfeead29845d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/4a4c5b964e6e4d1586ec05e3010f9905, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/053995d4d8a24d86b70936a56b06e943, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/f2c522d6d1e34dd99ea9631b6a3a871d] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=47.9 K 2024-11-07T15:23:17,477 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e3400c9047b4f5b98c8bfeead29845d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1730992992155 2024-11-07T15:23:17,478 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a4c5b964e6e4d1586ec05e3010f9905, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1730992992538 2024-11-07T15:23:17,478 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 053995d4d8a24d86b70936a56b06e943, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1730992993186 2024-11-07T15:23:17,478 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2c522d6d1e34dd99ea9631b6a3a871d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730992995372 2024-11-07T15:23:17,487 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#C#compaction#151 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:17,488 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/2087ecad3e1d41aa9082a2a4233a31e0 is 50, key is test_row_0/C:col10/1730992996511/Put/seqid=0 2024-11-07T15:23:17,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742008_1184 (size=12697) 2024-11-07T15:23:17,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T15:23:17,531 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:17,531 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:17,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-07T15:23:17,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T15:23:17,533 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:17,534 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:17,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:17,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T15:23:17,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T15:23:17,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:17,687 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-07T15:23:17,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110722576fd8919b4c93aec680547d5d8f90_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730992996518/Put/seqid=0 2024-11-07T15:23:17,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742009_1185 (size=12304) 2024-11-07T15:23:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:17,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:17,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993057752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:17,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:17,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993057754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:17,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:17,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:17,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:17,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993057754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993057756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:17,840 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/53ec97f1e3484c88ba735d6ecdcdb0a2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/53ec97f1e3484c88ba735d6ecdcdb0a2 2024-11-07T15:23:17,848 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/B of 518e68065b228feb1bbc2642f1a6f79d into 53ec97f1e3484c88ba735d6ecdcdb0a2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:17,848 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:17,848 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/B, priority=12, startTime=1730992997400; duration=0sec 2024-11-07T15:23:17,848 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:17,848 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:B 2024-11-07T15:23:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T15:23:17,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993057861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993057861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993057861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:17,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993057871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:17,899 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/2087ecad3e1d41aa9082a2a4233a31e0 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/2087ecad3e1d41aa9082a2a4233a31e0 2024-11-07T15:23:17,904 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/C of 518e68065b228feb1bbc2642f1a6f79d into 2087ecad3e1d41aa9082a2a4233a31e0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:17,904 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:17,904 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/C, priority=12, startTime=1730992997400; duration=0sec 2024-11-07T15:23:17,905 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:17,905 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:C 2024-11-07T15:23:18,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993058071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993058071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:18,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993058071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:18,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993058081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:18,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:18,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:18,103 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110722576fd8919b4c93aec680547d5d8f90_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110722576fd8919b4c93aec680547d5d8f90_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:18,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/88f95f04eded43e4bdbad52695d6aedd, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:18,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/88f95f04eded43e4bdbad52695d6aedd is 175, key is test_row_0/A:col10/1730992996518/Put/seqid=0 2024-11-07T15:23:18,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742010_1186 (size=31105) 2024-11-07T15:23:18,112 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/88f95f04eded43e4bdbad52695d6aedd 2024-11-07T15:23:18,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/706f1808abb14e1ba7edea84cd57dbf5 is 50, key is test_row_0/B:col10/1730992996518/Put/seqid=0 2024-11-07T15:23:18,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742011_1187 (size=12151) 2024-11-07T15:23:18,124 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/706f1808abb14e1ba7edea84cd57dbf5 2024-11-07T15:23:18,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/d1736c6387a7459687b74ee07723aa3b is 50, key is test_row_0/C:col10/1730992996518/Put/seqid=0 2024-11-07T15:23:18,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742012_1188 (size=12151) 2024-11-07T15:23:18,136 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/d1736c6387a7459687b74ee07723aa3b 2024-11-07T15:23:18,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/88f95f04eded43e4bdbad52695d6aedd as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/88f95f04eded43e4bdbad52695d6aedd 2024-11-07T15:23:18,146 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/88f95f04eded43e4bdbad52695d6aedd, entries=150, sequenceid=255, filesize=30.4 K 2024-11-07T15:23:18,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/706f1808abb14e1ba7edea84cd57dbf5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/706f1808abb14e1ba7edea84cd57dbf5 2024-11-07T15:23:18,152 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/706f1808abb14e1ba7edea84cd57dbf5, entries=150, sequenceid=255, filesize=11.9 K 2024-11-07T15:23:18,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/d1736c6387a7459687b74ee07723aa3b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/d1736c6387a7459687b74ee07723aa3b 2024-11-07T15:23:18,157 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/d1736c6387a7459687b74ee07723aa3b, entries=150, sequenceid=255, filesize=11.9 K 2024-11-07T15:23:18,158 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for 518e68065b228feb1bbc2642f1a6f79d in 471ms, sequenceid=255, compaction requested=false 2024-11-07T15:23:18,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:18,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:18,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-07T15:23:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-07T15:23:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T15:23:18,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-07T15:23:18,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 626 msec 2024-11-07T15:23:18,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 629 msec 2024-11-07T15:23:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:18,383 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:18,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-07T15:23:18,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:18,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993058383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993058383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:18,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993058385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:18,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993058391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:18,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077de6db62ce8c4a1581acc499ad78a226_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730992997750/Put/seqid=0 2024-11-07T15:23:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:18,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742013_1189 (size=14994) 2024-11-07T15:23:18,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:18,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993058491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993058491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:18,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:18,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993058501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:18,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T15:23:18,671 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:18,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-07T15:23:18,672 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T15:23:18,673 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:18,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:18,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993058701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:18,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993058701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:18,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993058711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:18,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T15:23:18,798 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:18,802 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411077de6db62ce8c4a1581acc499ad78a226_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077de6db62ce8c4a1581acc499ad78a226_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:18,803 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/7fccbdd5e7f64d45aeddd79411fb2b57, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:18,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/7fccbdd5e7f64d45aeddd79411fb2b57 is 175, key is test_row_0/A:col10/1730992997750/Put/seqid=0 2024-11-07T15:23:18,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742014_1190 (size=39949) 2024-11-07T15:23:18,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T15:23:18,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:18,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:18,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:18,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:18,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:18,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993058900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:18,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T15:23:18,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:18,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:18,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:18,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:18,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T15:23:19,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993059011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:19,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993059011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:19,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993059021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:19,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T15:23:19,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:19,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:19,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:19,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,208 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=64.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/7fccbdd5e7f64d45aeddd79411fb2b57 2024-11-07T15:23:19,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/647c2a7b10d34a139c1a509f024af601 is 50, key is test_row_0/B:col10/1730992997750/Put/seqid=0 2024-11-07T15:23:19,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742015_1191 (size=12301) 2024-11-07T15:23:19,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T15:23:19,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:19,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:19,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:19,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T15:23:19,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T15:23:19,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:19,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:19,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:19,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993059521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:19,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993059521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:19,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:19,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993059531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:19,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T15:23:19,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:19,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:19,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:19,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/647c2a7b10d34a139c1a509f024af601 2024-11-07T15:23:19,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/8926e7efec7244b3aefbd457e4788689 is 50, key is test_row_0/C:col10/1730992997750/Put/seqid=0 2024-11-07T15:23:19,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742016_1192 (size=12301) 2024-11-07T15:23:19,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T15:23:19,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:19,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:19,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:19,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T15:23:19,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T15:23:19,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:19,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:19,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:19,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:19,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993059911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:19,912 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_0' in 518e68065b228feb1bbc2642f1a6f79d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:23:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:20,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/8926e7efec7244b3aefbd457e4788689 2024-11-07T15:23:20,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/7fccbdd5e7f64d45aeddd79411fb2b57 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/7fccbdd5e7f64d45aeddd79411fb2b57 2024-11-07T15:23:20,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/7fccbdd5e7f64d45aeddd79411fb2b57, entries=200, sequenceid=289, filesize=39.0 K 2024-11-07T15:23:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/647c2a7b10d34a139c1a509f024af601 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/647c2a7b10d34a139c1a509f024af601 2024-11-07T15:23:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/647c2a7b10d34a139c1a509f024af601, entries=150, sequenceid=289, filesize=12.0 K 2024-11-07T15:23:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/8926e7efec7244b3aefbd457e4788689 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/8926e7efec7244b3aefbd457e4788689 2024-11-07T15:23:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T15:23:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:20,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:20,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:20,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:20,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/8926e7efec7244b3aefbd457e4788689, entries=150, sequenceid=289, filesize=12.0 K 2024-11-07T15:23:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=6.71 KB/6870 for 518e68065b228feb1bbc2642f1a6f79d in 1672ms, sequenceid=289, compaction requested=true 2024-11-07T15:23:20,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,056 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:20,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:20,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,056 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:20,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:20,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:20,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,057 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:20,057 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/B is initiating minor compaction (all files) 2024-11-07T15:23:20,057 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/B in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,058 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/53ec97f1e3484c88ba735d6ecdcdb0a2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/706f1808abb14e1ba7edea84cd57dbf5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/647c2a7b10d34a139c1a509f024af601] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=36.3 K 2024-11-07T15:23:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,058 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:20,058 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/A is initiating minor compaction (all files) 2024-11-07T15:23:20,058 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/A in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:20,058 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/f1af365f5c8d4536a8563d9cee00b579, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/88f95f04eded43e4bdbad52695d6aedd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/7fccbdd5e7f64d45aeddd79411fb2b57] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=100.3 K 2024-11-07T15:23:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,058 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:20,059 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/f1af365f5c8d4536a8563d9cee00b579, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/88f95f04eded43e4bdbad52695d6aedd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/7fccbdd5e7f64d45aeddd79411fb2b57] 2024-11-07T15:23:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,059 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 53ec97f1e3484c88ba735d6ecdcdb0a2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730992995372 2024-11-07T15:23:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,059 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 706f1808abb14e1ba7edea84cd57dbf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1730992996517 2024-11-07T15:23:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,060 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1af365f5c8d4536a8563d9cee00b579, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730992995372 2024-11-07T15:23:20,060 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 647c2a7b10d34a139c1a509f024af601, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730992997750 2024-11-07T15:23:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,061 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88f95f04eded43e4bdbad52695d6aedd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1730992996517 2024-11-07T15:23:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,061 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fccbdd5e7f64d45aeddd79411fb2b57, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730992997750 2024-11-07T15:23:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,074 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#B#compaction#158 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,075 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/b357accf36544931801311152ebc52c4 is 50, key is test_row_0/B:col10/1730992997750/Put/seqid=0 2024-11-07T15:23:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,076 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,093 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107ec498c3f0ddd4106a7e2a036f91d4012_518e68065b228feb1bbc2642f1a6f79d store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:20,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742017_1193 (size=12949) 2024-11-07T15:23:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,097 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107ec498c3f0ddd4106a7e2a036f91d4012_518e68065b228feb1bbc2642f1a6f79d, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,097 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ec498c3f0ddd4106a7e2a036f91d4012_518e68065b228feb1bbc2642f1a6f79d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742018_1194 (size=4469) 2024-11-07T15:23:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T15:23:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,203 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-07T15:23:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107459207d748154b32b3aa13cb0cf40df6_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_2/A:col10/1730992998383/Put/seqid=0 2024-11-07T15:23:20,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742019_1195 (size=7374) 2024-11-07T15:23:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,501 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/b357accf36544931801311152ebc52c4 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/b357accf36544931801311152ebc52c4 2024-11-07T15:23:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,507 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/B of 518e68065b228feb1bbc2642f1a6f79d into b357accf36544931801311152ebc52c4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:20,507 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,508 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/B, priority=13, startTime=1730993000056; duration=0sec 2024-11-07T15:23:20,508 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:20,508 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:B 2024-11-07T15:23:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,508 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,509 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:20,509 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/C is initiating minor compaction (all files) 2024-11-07T15:23:20,509 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/C in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,509 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/2087ecad3e1d41aa9082a2a4233a31e0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/d1736c6387a7459687b74ee07723aa3b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/8926e7efec7244b3aefbd457e4788689] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=36.3 K 2024-11-07T15:23:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,510 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 2087ecad3e1d41aa9082a2a4233a31e0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1730992995372 2024-11-07T15:23:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,510 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting d1736c6387a7459687b74ee07723aa3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1730992996517 2024-11-07T15:23:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,511 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 8926e7efec7244b3aefbd457e4788689, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730992997750 2024-11-07T15:23:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,519 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#C#compaction#161 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:20,520 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#A#compaction#159 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,520 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/1db9819d01d54a549b4e3f4c3babf6dd is 50, key is test_row_0/C:col10/1730992997750/Put/seqid=0 2024-11-07T15:23:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,520 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/506d9c52c43b47ba8b614e09a7573c11 is 175, key is test_row_0/A:col10/1730992997750/Put/seqid=0 2024-11-07T15:23:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742021_1197 (size=12949) 2024-11-07T15:23:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,538 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/1db9819d01d54a549b4e3f4c3babf6dd as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/1db9819d01d54a549b4e3f4c3babf6dd 2024-11-07T15:23:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,546 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/C of 518e68065b228feb1bbc2642f1a6f79d into 1db9819d01d54a549b4e3f4c3babf6dd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,546 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:20,546 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/C, priority=13, startTime=1730993000056; duration=0sec 2024-11-07T15:23:20,546 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,546 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:C 2024-11-07T15:23:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:20,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742020_1196 (size=31903) 2024-11-07T15:23:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,612 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/506d9c52c43b47ba8b614e09a7573c11 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/506d9c52c43b47ba8b614e09a7573c11 2024-11-07T15:23:20,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:20,625 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/A of 518e68065b228feb1bbc2642f1a6f79d into 506d9c52c43b47ba8b614e09a7573c11(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:20,626 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:20,626 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/A, priority=13, startTime=1730993000056; duration=0sec 2024-11-07T15:23:20,626 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:20,626 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:A 2024-11-07T15:23:20,629 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107459207d748154b32b3aa13cb0cf40df6_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107459207d748154b32b3aa13cb0cf40df6_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:20,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/a0c2da7d1efc478395c15fff3f61de75, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:20,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/a0c2da7d1efc478395c15fff3f61de75 is 175, key is test_row_2/A:col10/1730992998383/Put/seqid=0 2024-11-07T15:23:20,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993060637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:20,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742022_1198 (size=13865) 2024-11-07T15:23:20,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993060636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:20,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:20,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:20,645 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/a0c2da7d1efc478395c15fff3f61de75 2024-11-07T15:23:20,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993060643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:20,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:20,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/5a14735a7e044a9b9cdc86475416aaf3 is 50, key is test_row_2/B:col10/1730992998383/Put/seqid=0 2024-11-07T15:23:20,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742023_1199 (size=7415) 2024-11-07T15:23:20,683 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/5a14735a7e044a9b9cdc86475416aaf3 2024-11-07T15:23:20,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/19a22a7043a04bfbb3fbda25f2f4afc6 is 50, key is test_row_2/C:col10/1730992998383/Put/seqid=0 2024-11-07T15:23:20,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742024_1200 (size=7415) 2024-11-07T15:23:20,706 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/19a22a7043a04bfbb3fbda25f2f4afc6 2024-11-07T15:23:20,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/a0c2da7d1efc478395c15fff3f61de75 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/a0c2da7d1efc478395c15fff3f61de75 2024-11-07T15:23:20,722 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/a0c2da7d1efc478395c15fff3f61de75, entries=50, sequenceid=293, filesize=13.5 K 2024-11-07T15:23:20,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/5a14735a7e044a9b9cdc86475416aaf3 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/5a14735a7e044a9b9cdc86475416aaf3 2024-11-07T15:23:20,736 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/5a14735a7e044a9b9cdc86475416aaf3, entries=50, sequenceid=293, filesize=7.2 K 2024-11-07T15:23:20,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/19a22a7043a04bfbb3fbda25f2f4afc6 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/19a22a7043a04bfbb3fbda25f2f4afc6 2024-11-07T15:23:20,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1730993060751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:20,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993060751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:20,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993060751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:20,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993060751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:20,756 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/19a22a7043a04bfbb3fbda25f2f4afc6, entries=50, sequenceid=293, filesize=7.2 K 2024-11-07T15:23:20,756 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_0' in 518e68065b228feb1bbc2642f1a6f79d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:23:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:20,757 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=194.56 KB/199230 for 518e68065b228feb1bbc2642f1a6f79d in 554ms, sequenceid=293, compaction requested=false 2024-11-07T15:23:20,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:20,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:20,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-07T15:23:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-07T15:23:20,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-07T15:23:20,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0860 sec 2024-11-07T15:23:20,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.0920 sec 2024-11-07T15:23:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T15:23:20,821 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:20,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-07T15:23:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T15:23:20,823 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:20,824 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:20,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T15:23:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:20,964 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:20,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=201.27 KB heapSize=528.09 KB 2024-11-07T15:23:20,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:20,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993060961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993060961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:20,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993060965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:20,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071fa6e87567414a2ea1027e159f9b246b_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730993000633/Put/seqid=0 2024-11-07T15:23:20,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:23:20,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:20,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:20,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:20,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:20,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:20,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742025_1201 (size=12454) 2024-11-07T15:23:21,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993061071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:21,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:21,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:21,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:21,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:23:21,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:21,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:21,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:21,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:21,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T15:23:21,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993061270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:21,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993061270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:21,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:21,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993061281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:21,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:23:21,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:21,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:21,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:21,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:21,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:21,381 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,385 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071fa6e87567414a2ea1027e159f9b246b_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071fa6e87567414a2ea1027e159f9b246b_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:21,386 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/ae60e671ead24678b0804fbcbf76d175, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:21,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/ae60e671ead24678b0804fbcbf76d175 is 175, key is test_row_0/A:col10/1730993000633/Put/seqid=0 2024-11-07T15:23:21,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742026_1202 (size=31255) 2024-11-07T15:23:21,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:23:21,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:21,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:21,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:21,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:21,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T15:23:21,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:23:21,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:21,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:21,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:21,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:21,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:21,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993061590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:21,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:23:21,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:21,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:21,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:21,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:21,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:21,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993061781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:21,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993061781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:21,791 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=67.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/ae60e671ead24678b0804fbcbf76d175 2024-11-07T15:23:21,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/1cbeeae4ac7344c4883537680ece9575 is 50, key is test_row_0/B:col10/1730993000633/Put/seqid=0 2024-11-07T15:23:21,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742027_1203 (size=12301) 2024-11-07T15:23:21,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/1cbeeae4ac7344c4883537680ece9575 2024-11-07T15:23:21,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/93e1180ec03e4ae997f6e733c816e623 is 50, key is test_row_0/C:col10/1730993000633/Put/seqid=0 2024-11-07T15:23:21,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742028_1204 (size=12301) 2024-11-07T15:23:21,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/93e1180ec03e4ae997f6e733c816e623 2024-11-07T15:23:21,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/ae60e671ead24678b0804fbcbf76d175 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ae60e671ead24678b0804fbcbf76d175 2024-11-07T15:23:21,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ae60e671ead24678b0804fbcbf76d175, entries=150, sequenceid=329, filesize=30.5 K 2024-11-07T15:23:21,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/1cbeeae4ac7344c4883537680ece9575 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/1cbeeae4ac7344c4883537680ece9575 2024-11-07T15:23:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/1cbeeae4ac7344c4883537680ece9575, entries=150, sequenceid=329, filesize=12.0 K 2024-11-07T15:23:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/93e1180ec03e4ae997f6e733c816e623 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/93e1180ec03e4ae997f6e733c816e623 2024-11-07T15:23:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/93e1180ec03e4ae997f6e733c816e623, entries=150, sequenceid=329, filesize=12.0 K 2024-11-07T15:23:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for 518e68065b228feb1bbc2642f1a6f79d in 880ms, sequenceid=329, compaction requested=true 2024-11-07T15:23:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:21,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:21,845 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:21,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:21,845 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:21,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:21,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:21,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,846 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:21,846 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 77023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:21,846 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/B is initiating minor compaction (all files) 2024-11-07T15:23:21,846 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/A is initiating minor compaction (all files) 2024-11-07T15:23:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,846 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/B in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:21,846 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/A in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:21,847 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/b357accf36544931801311152ebc52c4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/5a14735a7e044a9b9cdc86475416aaf3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/1cbeeae4ac7344c4883537680ece9575] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=31.9 K 2024-11-07T15:23:21,847 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/506d9c52c43b47ba8b614e09a7573c11, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/a0c2da7d1efc478395c15fff3f61de75, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ae60e671ead24678b0804fbcbf76d175] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=75.2 K 2024-11-07T15:23:21,847 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,847 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/506d9c52c43b47ba8b614e09a7573c11, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/a0c2da7d1efc478395c15fff3f61de75, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ae60e671ead24678b0804fbcbf76d175] 2024-11-07T15:23:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,847 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting b357accf36544931801311152ebc52c4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730992997750 2024-11-07T15:23:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,848 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a14735a7e044a9b9cdc86475416aaf3, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1730992998383 2024-11-07T15:23:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,848 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 506d9c52c43b47ba8b614e09a7573c11, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730992997750 2024-11-07T15:23:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,848 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cbeeae4ac7344c4883537680ece9575, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1730993000633 2024-11-07T15:23:21,848 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0c2da7d1efc478395c15fff3f61de75, keycount=50, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1730992998383 2024-11-07T15:23:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,849 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae60e671ead24678b0804fbcbf76d175, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1730993000633 2024-11-07T15:23:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,858 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,870 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,871 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/e3df2789ef254a239401fbe2528fece1 is 50, key is test_row_0/B:col10/1730993000633/Put/seqid=0 2024-11-07T15:23:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,872 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110719c909fbbc6b4c538a1c9f31683a09e7_518e68065b228feb1bbc2642f1a6f79d store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,874 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110719c909fbbc6b4c538a1c9f31683a09e7_518e68065b228feb1bbc2642f1a6f79d, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,874 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110719c909fbbc6b4c538a1c9f31683a09e7_518e68065b228feb1bbc2642f1a6f79d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:23:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:21,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:21,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:21,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-07T15:23:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-07T15:23:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742029_1205 (size=13051) 2024-11-07T15:23:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-07T15:23:21,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0720 sec 2024-11-07T15:23:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.0790 sec 2024-11-07T15:23:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,903 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/e3df2789ef254a239401fbe2528fece1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/e3df2789ef254a239401fbe2528fece1 2024-11-07T15:23:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,910 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/B of 518e68065b228feb1bbc2642f1a6f79d into e3df2789ef254a239401fbe2528fece1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:21,910 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:21,910 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/B, priority=13, startTime=1730993001845; duration=0sec 2024-11-07T15:23:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,910 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:21,910 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:B 2024-11-07T15:23:21,910 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,912 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:21,912 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/C is initiating minor compaction (all files) 2024-11-07T15:23:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,913 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/C in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:21,913 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/1db9819d01d54a549b4e3f4c3babf6dd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/19a22a7043a04bfbb3fbda25f2f4afc6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/93e1180ec03e4ae997f6e733c816e623] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=31.9 K 2024-11-07T15:23:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742030_1206 (size=4469) 2024-11-07T15:23:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,915 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 1db9819d01d54a549b4e3f4c3babf6dd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730992997750 2024-11-07T15:23:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,916 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 19a22a7043a04bfbb3fbda25f2f4afc6, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1730992998383 2024-11-07T15:23:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,916 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#A#compaction#167 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,917 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 93e1180ec03e4ae997f6e733c816e623, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1730993000633 2024-11-07T15:23:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,917 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/39086b5ef7344dd9b97fa898ddbd6d65 is 175, key is test_row_0/A:col10/1730993000633/Put/seqid=0 2024-11-07T15:23:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742031_1207 (size=32005) 2024-11-07T15:23:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,929 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/39086b5ef7344dd9b97fa898ddbd6d65 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/39086b5ef7344dd9b97fa898ddbd6d65 2024-11-07T15:23:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,930 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#C#compaction#169 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,930 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/549957c874834fc093dff65171194e9f is 50, key is test_row_0/C:col10/1730993000633/Put/seqid=0 2024-11-07T15:23:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,938 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/A of 518e68065b228feb1bbc2642f1a6f79d into 39086b5ef7344dd9b97fa898ddbd6d65(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:21,938 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:21,938 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/A, priority=13, startTime=1730993001844; duration=0sec 2024-11-07T15:23:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,938 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:21,938 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:A 2024-11-07T15:23:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742032_1208 (size=13051) 2024-11-07T15:23:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T15:23:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,961 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,961 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-07T15:23:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T15:23:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,963 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,964 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:21,999 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:22,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:23:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110709eca3a6961246f19c4587c4598bcf39_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730993001998/Put/seqid=0 2024-11-07T15:23:22,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742034_1210 (size=25158) 2024-11-07T15:23:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T15:23:22,071 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,077 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110709eca3a6961246f19c4587c4598bcf39_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110709eca3a6961246f19c4587c4598bcf39_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:22,079 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/8eda66eac6ac46eaa0f3066c90eba450, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:22,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/8eda66eac6ac46eaa0f3066c90eba450 is 175, key is test_row_0/A:col10/1730993001998/Put/seqid=0 2024-11-07T15:23:22,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742033_1209 (size=74795) 2024-11-07T15:23:22,083 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/8eda66eac6ac46eaa0f3066c90eba450 2024-11-07T15:23:22,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/0ecc40212a9a4daa8d36ddaf367f91c1 is 50, key is test_row_0/B:col10/1730993001998/Put/seqid=0 2024-11-07T15:23:22,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993062101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:22,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993062103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:22,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T15:23:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:22,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742035_1211 (size=12301) 2024-11-07T15:23:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/0ecc40212a9a4daa8d36ddaf367f91c1 2024-11-07T15:23:22,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/6a60d3332d9b49f78449fa7d496fc578 is 50, key is test_row_0/C:col10/1730993001998/Put/seqid=0 2024-11-07T15:23:22,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742036_1212 (size=12301) 2024-11-07T15:23:22,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/6a60d3332d9b49f78449fa7d496fc578 2024-11-07T15:23:22,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/8eda66eac6ac46eaa0f3066c90eba450 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/8eda66eac6ac46eaa0f3066c90eba450 2024-11-07T15:23:22,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/8eda66eac6ac46eaa0f3066c90eba450, entries=400, sequenceid=343, filesize=73.0 K 2024-11-07T15:23:22,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/0ecc40212a9a4daa8d36ddaf367f91c1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/0ecc40212a9a4daa8d36ddaf367f91c1 2024-11-07T15:23:22,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/0ecc40212a9a4daa8d36ddaf367f91c1, entries=150, sequenceid=343, filesize=12.0 K 2024-11-07T15:23:22,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/6a60d3332d9b49f78449fa7d496fc578 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/6a60d3332d9b49f78449fa7d496fc578 2024-11-07T15:23:22,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/6a60d3332d9b49f78449fa7d496fc578, entries=150, sequenceid=343, filesize=12.0 K 2024-11-07T15:23:22,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 518e68065b228feb1bbc2642f1a6f79d in 184ms, sequenceid=343, compaction requested=false 2024-11-07T15:23:22,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:22,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T15:23:22,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cd086bf149a042578a92d0f878072e8a_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730993002096/Put/seqid=0 2024-11-07T15:23:22,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742037_1213 (size=14994) 2024-11-07T15:23:22,256 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:22,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T15:23:22,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:22,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:22,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:22,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T15:23:22,303 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cd086bf149a042578a92d0f878072e8a_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cd086bf149a042578a92d0f878072e8a_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:22,305 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/0c9f42894b6a4151a9b52e283e158a59, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:22,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/0c9f42894b6a4151a9b52e283e158a59 is 175, key is test_row_0/A:col10/1730993002096/Put/seqid=0 2024-11-07T15:23:22,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742038_1214 (size=39949) 2024-11-07T15:23:22,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993062309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:22,348 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/549957c874834fc093dff65171194e9f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/549957c874834fc093dff65171194e9f 2024-11-07T15:23:22,355 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/C of 518e68065b228feb1bbc2642f1a6f79d into 549957c874834fc093dff65171194e9f(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:22,355 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:22,355 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/C, priority=13, startTime=1730993001845; duration=0sec 2024-11-07T15:23:22,355 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:22,355 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:C 2024-11-07T15:23:22,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993062420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:22,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T15:23:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:22,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T15:23:22,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T15:23:22,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:22,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:22,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:22,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993062631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:22,712 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/0c9f42894b6a4151a9b52e283e158a59 2024-11-07T15:23:22,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/f76797bad7124a72b193383acdc2e2cd is 50, key is test_row_0/B:col10/1730993002096/Put/seqid=0 2024-11-07T15:23:22,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742039_1215 (size=12301) 2024-11-07T15:23:22,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T15:23:22,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:22,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:22,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:22,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993062790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:22,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993062790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:22,793 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in 518e68065b228feb1bbc2642f1a6f79d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:23:22,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:22,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:22,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:22,794 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in 518e68065b228feb1bbc2642f1a6f79d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:23:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:22,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T15:23:22,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:22,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:22,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:22,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:22,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993062941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:22,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:22,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:22,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:23,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T15:23:23,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:23,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:23,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:23,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:23,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T15:23:23,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993063111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:23,117 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in 518e68065b228feb1bbc2642f1a6f79d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:23:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:23,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/f76797bad7124a72b193383acdc2e2cd 2024-11-07T15:23:23,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/e479de42f2a6427aa56326cfac86bc70 is 50, key is test_row_0/C:col10/1730993002096/Put/seqid=0 2024-11-07T15:23:23,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742040_1216 (size=12301) 2024-11-07T15:23:23,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T15:23:23,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:23,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:23,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:23,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:23,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:23,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T15:23:23,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:23,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:23,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:23,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:23,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:23,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993063451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:23,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T15:23:23,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:23,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:23,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:23,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:23,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:23,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/e479de42f2a6427aa56326cfac86bc70 2024-11-07T15:23:23,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/0c9f42894b6a4151a9b52e283e158a59 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/0c9f42894b6a4151a9b52e283e158a59 2024-11-07T15:23:23,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/0c9f42894b6a4151a9b52e283e158a59, entries=200, sequenceid=369, filesize=39.0 K 2024-11-07T15:23:23,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/f76797bad7124a72b193383acdc2e2cd as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/f76797bad7124a72b193383acdc2e2cd 2024-11-07T15:23:23,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/f76797bad7124a72b193383acdc2e2cd, entries=150, sequenceid=369, filesize=12.0 K 2024-11-07T15:23:23,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/e479de42f2a6427aa56326cfac86bc70 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/e479de42f2a6427aa56326cfac86bc70 2024-11-07T15:23:23,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/e479de42f2a6427aa56326cfac86bc70, entries=150, sequenceid=369, filesize=12.0 K 2024-11-07T15:23:23,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 518e68065b228feb1bbc2642f1a6f79d in 1352ms, sequenceid=369, compaction requested=true 2024-11-07T15:23:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:23,567 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:23,567 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:23,569 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:23,569 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/B is initiating minor compaction (all files) 2024-11-07T15:23:23,569 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/B in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:23,569 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/e3df2789ef254a239401fbe2528fece1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/0ecc40212a9a4daa8d36ddaf367f91c1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/f76797bad7124a72b193383acdc2e2cd] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=36.8 K 2024-11-07T15:23:23,569 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting e3df2789ef254a239401fbe2528fece1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1730993000633 2024-11-07T15:23:23,570 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 146749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:23,570 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/A is initiating minor compaction (all files) 2024-11-07T15:23:23,570 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/A in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:23,570 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/39086b5ef7344dd9b97fa898ddbd6d65, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/8eda66eac6ac46eaa0f3066c90eba450, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/0c9f42894b6a4151a9b52e283e158a59] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=143.3 K 2024-11-07T15:23:23,570 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:23,570 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/39086b5ef7344dd9b97fa898ddbd6d65, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/8eda66eac6ac46eaa0f3066c90eba450, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/0c9f42894b6a4151a9b52e283e158a59] 2024-11-07T15:23:23,570 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ecc40212a9a4daa8d36ddaf367f91c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1730993001985 2024-11-07T15:23:23,571 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39086b5ef7344dd9b97fa898ddbd6d65, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1730993000633 2024-11-07T15:23:23,571 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting f76797bad7124a72b193383acdc2e2cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1730993002096 2024-11-07T15:23:23,571 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eda66eac6ac46eaa0f3066c90eba450, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1730993001947 2024-11-07T15:23:23,572 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c9f42894b6a4151a9b52e283e158a59, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1730993002086 2024-11-07T15:23:23,578 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:23,582 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#B#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:23,583 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/52c9b38fe243410288ec7b4b91e75915 is 50, key is test_row_0/B:col10/1730993002096/Put/seqid=0 2024-11-07T15:23:23,587 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107fbcfd6485c4a447194d7eec20f492b93_518e68065b228feb1bbc2642f1a6f79d store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:23,590 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107fbcfd6485c4a447194d7eec20f492b93_518e68065b228feb1bbc2642f1a6f79d, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:23,590 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107fbcfd6485c4a447194d7eec20f492b93_518e68065b228feb1bbc2642f1a6f79d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:23,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742041_1217 (size=13153) 2024-11-07T15:23:23,617 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/52c9b38fe243410288ec7b4b91e75915 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/52c9b38fe243410288ec7b4b91e75915 2024-11-07T15:23:23,623 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/B of 518e68065b228feb1bbc2642f1a6f79d into 52c9b38fe243410288ec7b4b91e75915(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:23,623 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:23,623 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/B, priority=13, startTime=1730993003567; duration=0sec 2024-11-07T15:23:23,623 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:23,623 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:B 2024-11-07T15:23:23,623 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:23,624 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:23,625 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/C is initiating minor compaction (all files) 2024-11-07T15:23:23,625 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/C in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:23,625 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/549957c874834fc093dff65171194e9f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/6a60d3332d9b49f78449fa7d496fc578, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/e479de42f2a6427aa56326cfac86bc70] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=36.8 K 2024-11-07T15:23:23,625 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 549957c874834fc093dff65171194e9f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1730993000633 2024-11-07T15:23:23,626 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a60d3332d9b49f78449fa7d496fc578, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1730993001985 2024-11-07T15:23:23,626 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting e479de42f2a6427aa56326cfac86bc70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1730993002096 2024-11-07T15:23:23,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742042_1218 (size=4469) 2024-11-07T15:23:23,635 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#A#compaction#176 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:23,636 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/b024e0d9098b4835ba4736c3c4be493e is 175, key is test_row_0/A:col10/1730993002096/Put/seqid=0 2024-11-07T15:23:23,636 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#C#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:23,637 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/b3449353f6f14d38ba014b363609f300 is 50, key is test_row_0/C:col10/1730993002096/Put/seqid=0 2024-11-07T15:23:23,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742043_1219 (size=32107) 2024-11-07T15:23:23,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742044_1220 (size=13153) 2024-11-07T15:23:23,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T15:23:23,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:23,676 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T15:23:23,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075d5de20648334cdf988af05860fc2d3e_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730993002253/Put/seqid=0 2024-11-07T15:23:23,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742045_1221 (size=12454) 2024-11-07T15:23:24,072 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/b3449353f6f14d38ba014b363609f300 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/b3449353f6f14d38ba014b363609f300 2024-11-07T15:23:24,075 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/b024e0d9098b4835ba4736c3c4be493e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/b024e0d9098b4835ba4736c3c4be493e 2024-11-07T15:23:24,080 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/C of 518e68065b228feb1bbc2642f1a6f79d into b3449353f6f14d38ba014b363609f300(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:24,080 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:24,080 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/C, priority=13, startTime=1730993003567; duration=0sec 2024-11-07T15:23:24,081 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:24,081 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:C 2024-11-07T15:23:24,082 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/A of 518e68065b228feb1bbc2642f1a6f79d into b024e0d9098b4835ba4736c3c4be493e(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:24,082 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:24,082 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/A, priority=13, startTime=1730993003567; duration=0sec 2024-11-07T15:23:24,082 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:24,082 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:A 2024-11-07T15:23:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:24,103 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075d5de20648334cdf988af05860fc2d3e_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075d5de20648334cdf988af05860fc2d3e_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:24,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/503284085ff246dfa7037ed0673c35f1, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:24,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/503284085ff246dfa7037ed0673c35f1 is 175, key is test_row_0/A:col10/1730993002253/Put/seqid=0 2024-11-07T15:23:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T15:23:24,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742046_1222 (size=31255) 2024-11-07T15:23:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:24,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:24,528 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=381, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/503284085ff246dfa7037ed0673c35f1 2024-11-07T15:23:24,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/3270b9c259214b0da41899a7e23bdcd5 is 50, key is test_row_0/B:col10/1730993002253/Put/seqid=0 2024-11-07T15:23:24,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742047_1223 (size=12301) 2024-11-07T15:23:24,551 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/3270b9c259214b0da41899a7e23bdcd5 2024-11-07T15:23:24,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/eee22a1ac78545f0b3e2b48ad844284c is 50, key is test_row_0/C:col10/1730993002253/Put/seqid=0 2024-11-07T15:23:24,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742048_1224 (size=12301) 2024-11-07T15:23:24,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993064579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:24,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:24,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:24,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993064690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:24,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:24,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:24,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:24,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38024 deadline: 1730993064800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:24,804 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in 518e68065b228feb1bbc2642f1a6f79d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:23:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:24,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38086 deadline: 1730993064811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:24,815 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in 518e68065b228feb1bbc2642f1a6f79d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:23:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:24,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:24,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993064900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:24,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:24,967 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/eee22a1ac78545f0b3e2b48ad844284c 2024-11-07T15:23:24,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/503284085ff246dfa7037ed0673c35f1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/503284085ff246dfa7037ed0673c35f1 2024-11-07T15:23:24,978 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/503284085ff246dfa7037ed0673c35f1, entries=150, sequenceid=381, filesize=30.5 K 2024-11-07T15:23:24,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/3270b9c259214b0da41899a7e23bdcd5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/3270b9c259214b0da41899a7e23bdcd5 2024-11-07T15:23:24,983 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/3270b9c259214b0da41899a7e23bdcd5, entries=150, sequenceid=381, filesize=12.0 K 2024-11-07T15:23:24,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/eee22a1ac78545f0b3e2b48ad844284c as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/eee22a1ac78545f0b3e2b48ad844284c 2024-11-07T15:23:24,992 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/eee22a1ac78545f0b3e2b48ad844284c, entries=150, sequenceid=381, filesize=12.0 K 2024-11-07T15:23:24,993 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 518e68065b228feb1bbc2642f1a6f79d in 1317ms, sequenceid=381, compaction requested=false 2024-11-07T15:23:24,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:24,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:24,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-07T15:23:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-07T15:23:24,997 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-07T15:23:24,997 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0310 sec 2024-11-07T15:23:24,999 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 3.0370 sec 2024-11-07T15:23:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:25,133 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:25,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T15:23:25,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074ca1d206ef0e47dc9c0627c0d4060281_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730993004524/Put/seqid=0 2024-11-07T15:23:25,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742049_1225 (size=14994) 2024-11-07T15:23:25,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:25,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993065151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:25,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:25,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:25,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993065211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:25,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:25,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993065261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:25,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:25,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:25,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:25,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993065471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:25,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:25,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:25,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:25,549 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,554 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074ca1d206ef0e47dc9c0627c0d4060281_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074ca1d206ef0e47dc9c0627c0d4060281_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:25,555 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/56f759e18ff84a86be233503d7434b27, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:25,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/56f759e18ff84a86be233503d7434b27 is 175, key is test_row_0/A:col10/1730993004524/Put/seqid=0 2024-11-07T15:23:25,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742050_1226 (size=39949) 2024-11-07T15:23:25,566 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/56f759e18ff84a86be233503d7434b27 2024-11-07T15:23:25,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/7714af2b38644450b810934c3a9e9f20 is 50, key is test_row_0/B:col10/1730993004524/Put/seqid=0 2024-11-07T15:23:25,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742051_1227 (size=12301) 2024-11-07T15:23:25,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/7714af2b38644450b810934c3a9e9f20 2024-11-07T15:23:25,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/a0149618311440729b5041590c1f45ac is 50, key is test_row_0/C:col10/1730993004524/Put/seqid=0 2024-11-07T15:23:25,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742052_1228 (size=12301) 2024-11-07T15:23:25,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/a0149618311440729b5041590c1f45ac 2024-11-07T15:23:25,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/56f759e18ff84a86be233503d7434b27 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/56f759e18ff84a86be233503d7434b27 2024-11-07T15:23:25,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/56f759e18ff84a86be233503d7434b27, entries=200, sequenceid=410, filesize=39.0 K 2024-11-07T15:23:25,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/7714af2b38644450b810934c3a9e9f20 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/7714af2b38644450b810934c3a9e9f20 2024-11-07T15:23:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/7714af2b38644450b810934c3a9e9f20, entries=150, sequenceid=410, filesize=12.0 K 2024-11-07T15:23:25,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/a0149618311440729b5041590c1f45ac as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/a0149618311440729b5041590c1f45ac 2024-11-07T15:23:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/a0149618311440729b5041590c1f45ac, entries=150, sequenceid=410, filesize=12.0 K 2024-11-07T15:23:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 518e68065b228feb1bbc2642f1a6f79d in 560ms, sequenceid=410, compaction requested=true 2024-11-07T15:23:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,694 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:25,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:25,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:25,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 518e68065b228feb1bbc2642f1a6f79d:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:25,694 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:25,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,696 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:25,696 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/A is initiating minor compaction (all files) 2024-11-07T15:23:25,696 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/A in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,696 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/b024e0d9098b4835ba4736c3c4be493e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/503284085ff246dfa7037ed0673c35f1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/56f759e18ff84a86be233503d7434b27] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=100.9 K 2024-11-07T15:23:25,696 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:25,696 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/b024e0d9098b4835ba4736c3c4be493e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/503284085ff246dfa7037ed0673c35f1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/56f759e18ff84a86be233503d7434b27] 2024-11-07T15:23:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,697 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting b024e0d9098b4835ba4736c3c4be493e, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1730993002096 2024-11-07T15:23:25,698 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:25,698 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/B is initiating minor compaction (all files) 2024-11-07T15:23:25,698 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/B in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:25,698 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/52c9b38fe243410288ec7b4b91e75915, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/3270b9c259214b0da41899a7e23bdcd5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/7714af2b38644450b810934c3a9e9f20] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=36.9 K 2024-11-07T15:23:25,699 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 503284085ff246dfa7037ed0673c35f1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1730993002248 2024-11-07T15:23:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,699 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 52c9b38fe243410288ec7b4b91e75915, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1730993002096 2024-11-07T15:23:25,699 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56f759e18ff84a86be233503d7434b27, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1730993004524 2024-11-07T15:23:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,700 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 3270b9c259214b0da41899a7e23bdcd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1730993002248 2024-11-07T15:23:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,701 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 7714af2b38644450b810934c3a9e9f20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1730993004524 2024-11-07T15:23:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,718 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#B#compaction#185 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,719 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/6eaba82e3fd942b6bb31f74d2fd36d37 is 50, key is test_row_0/B:col10/1730993004524/Put/seqid=0 2024-11-07T15:23:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,724 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,727 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107f1287f07003c45cdaf1c20341a4ac673_518e68065b228feb1bbc2642f1a6f79d store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:25,728 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107f1287f07003c45cdaf1c20341a4ac673_518e68065b228feb1bbc2642f1a6f79d, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:25,729 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f1287f07003c45cdaf1c20341a4ac673_518e68065b228feb1bbc2642f1a6f79d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,745 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:23:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c6308084d7f3489b82d2049df09af4e7_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_1/A:col10/1730993005744/Put/seqid=0 2024-11-07T15:23:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742053_1229 (size=13255) 2024-11-07T15:23:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742054_1230 (size=4469) 2024-11-07T15:23:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742055_1231 (size=14994) 2024-11-07T15:23:25,839 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,847 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c6308084d7f3489b82d2049df09af4e7_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c6308084d7f3489b82d2049df09af4e7_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:25,848 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/ab6dd47c1d5f452ba011b7065d111001, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:25,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/ab6dd47c1d5f452ba011b7065d111001 is 175, key is test_row_1/A:col10/1730993005744/Put/seqid=0 2024-11-07T15:23:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:25,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742056_1232 (size=39945) 2024-11-07T15:23:25,875 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=421, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/ab6dd47c1d5f452ba011b7065d111001 2024-11-07T15:23:25,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/bac9f77ba9e444659fcdfb7fecd7a05f is 50, key is test_row_1/B:col10/1730993005744/Put/seqid=0 2024-11-07T15:23:25,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:25,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993065934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:25,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:25,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993065934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:25,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742057_1233 (size=9857) 2024-11-07T15:23:26,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993066050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:26,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:26,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:26,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:26,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993066050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T15:23:26,122 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:26,123 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-07T15:23:26,126 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T15:23:26,127 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:26,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:26,211 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/6eaba82e3fd942b6bb31f74d2fd36d37 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/6eaba82e3fd942b6bb31f74d2fd36d37 2024-11-07T15:23:26,218 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/B of 518e68065b228feb1bbc2642f1a6f79d into 6eaba82e3fd942b6bb31f74d2fd36d37(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:26,218 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:26,218 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/B, priority=13, startTime=1730993005694; duration=0sec 2024-11-07T15:23:26,218 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:26,218 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:B 2024-11-07T15:23:26,218 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:26,220 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:26,220 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 518e68065b228feb1bbc2642f1a6f79d/C is initiating minor compaction (all files) 2024-11-07T15:23:26,220 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 518e68065b228feb1bbc2642f1a6f79d/C in TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:26,220 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/b3449353f6f14d38ba014b363609f300, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/eee22a1ac78545f0b3e2b48ad844284c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/a0149618311440729b5041590c1f45ac] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp, totalSize=36.9 K 2024-11-07T15:23:26,220 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#A#compaction#186 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:26,220 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting b3449353f6f14d38ba014b363609f300, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1730993002096 2024-11-07T15:23:26,221 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting eee22a1ac78545f0b3e2b48ad844284c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1730993002248 2024-11-07T15:23:26,221 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/34c41dbae860485298dd0b7cedfb0402 is 175, key is test_row_0/A:col10/1730993004524/Put/seqid=0 2024-11-07T15:23:26,221 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting a0149618311440729b5041590c1f45ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1730993004524 2024-11-07T15:23:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T15:23:26,266 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 518e68065b228feb1bbc2642f1a6f79d#C#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:26,267 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/3ec92f4ea34b4e778c860a87638ba479 is 50, key is test_row_0/C:col10/1730993004524/Put/seqid=0 2024-11-07T15:23:26,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:26,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993066262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993066262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:26,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742058_1234 (size=32209) 2024-11-07T15:23:26,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T15:23:26,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:26,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:26,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:26,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:26,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:26,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742059_1235 (size=13255) 2024-11-07T15:23:26,335 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/3ec92f4ea34b4e778c860a87638ba479 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/3ec92f4ea34b4e778c860a87638ba479 2024-11-07T15:23:26,342 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/C of 518e68065b228feb1bbc2642f1a6f79d into 3ec92f4ea34b4e778c860a87638ba479(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:26,342 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:26,342 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/C, priority=13, startTime=1730993005694; duration=0sec 2024-11-07T15:23:26,342 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:26,342 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:C 2024-11-07T15:23:26,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/bac9f77ba9e444659fcdfb7fecd7a05f 2024-11-07T15:23:26,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/e3ff8f2f75bf4d93b444670819a60e55 is 50, key is test_row_1/C:col10/1730993005744/Put/seqid=0 2024-11-07T15:23:26,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742060_1236 (size=9857) 2024-11-07T15:23:26,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T15:23:26,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:26,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:26,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:26,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:26,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T15:23:26,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38022 deadline: 1730993066581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:26,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38062 deadline: 1730993066581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:23:26,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T15:23:26,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:26,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:26,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:26,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:26,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:26,687 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/34c41dbae860485298dd0b7cedfb0402 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/34c41dbae860485298dd0b7cedfb0402 2024-11-07T15:23:26,693 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 518e68065b228feb1bbc2642f1a6f79d/A of 518e68065b228feb1bbc2642f1a6f79d into 34c41dbae860485298dd0b7cedfb0402(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:26,693 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:26,693 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., storeName=518e68065b228feb1bbc2642f1a6f79d/A, priority=13, startTime=1730993005694; duration=0sec 2024-11-07T15:23:26,694 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:26,694 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 518e68065b228feb1bbc2642f1a6f79d:A 2024-11-07T15:23:26,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T15:23:26,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:26,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:26,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:26,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:26,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T15:23:26,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/e3ff8f2f75bf4d93b444670819a60e55 2024-11-07T15:23:26,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/ab6dd47c1d5f452ba011b7065d111001 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ab6dd47c1d5f452ba011b7065d111001 2024-11-07T15:23:26,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ab6dd47c1d5f452ba011b7065d111001, entries=200, sequenceid=421, filesize=39.0 K 2024-11-07T15:23:26,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/bac9f77ba9e444659fcdfb7fecd7a05f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/bac9f77ba9e444659fcdfb7fecd7a05f 2024-11-07T15:23:26,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/bac9f77ba9e444659fcdfb7fecd7a05f, entries=100, sequenceid=421, filesize=9.6 K 2024-11-07T15:23:26,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/e3ff8f2f75bf4d93b444670819a60e55 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/e3ff8f2f75bf4d93b444670819a60e55 2024-11-07T15:23:26,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/e3ff8f2f75bf4d93b444670819a60e55, entries=100, sequenceid=421, filesize=9.6 K 2024-11-07T15:23:26,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 518e68065b228feb1bbc2642f1a6f79d in 1096ms, sequenceid=421, compaction requested=false 2024-11-07T15:23:26,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:26,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T15:23:26,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:26,895 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T15:23:26,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076154f8fd202d4e929e9d77b0758eaf47_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730993005928/Put/seqid=0 2024-11-07T15:23:26,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742061_1237 (size=12454) 2024-11-07T15:23:26,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:26,938 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076154f8fd202d4e929e9d77b0758eaf47_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076154f8fd202d4e929e9d77b0758eaf47_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/cf5acdd558094d93a46a784a4ac4b9c2, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:26,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/cf5acdd558094d93a46a784a4ac4b9c2 is 175, key is test_row_0/A:col10/1730993005928/Put/seqid=0 2024-11-07T15:23:26,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742062_1238 (size=31255) 2024-11-07T15:23:26,954 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=449, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/cf5acdd558094d93a46a784a4ac4b9c2 2024-11-07T15:23:26,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/bb1eaa2cfc59496f829d3c96562549fc is 50, key is test_row_0/B:col10/1730993005928/Put/seqid=0 2024-11-07T15:23:26,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742063_1239 (size=12301) 2024-11-07T15:23:26,982 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/bb1eaa2cfc59496f829d3c96562549fc 2024-11-07T15:23:26,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/553ccae1b9cd4af687752fca7a813ea6 is 50, key is test_row_0/C:col10/1730993005928/Put/seqid=0 2024-11-07T15:23:27,027 INFO [Thread-607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-607. 2024-11-07T15:23:27,027 DEBUG [Thread-607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:27,027 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:27,027 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:27,027 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:27,030 INFO [Thread-609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-609. 2024-11-07T15:23:27,030 DEBUG [Thread-609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:27,030 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:27,030 INFO [Thread-613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-613. 2024-11-07T15:23:27,030 DEBUG [Thread-613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:27,030 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:27,030 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:27,030 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:27,030 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:27,030 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:27,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742064_1240 (size=12301) 2024-11-07T15:23:27,035 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-11-07T15:23:27,035 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:27,035 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:27,035 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:27,035 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:27,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:27,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. as already flushing 2024-11-07T15:23:27,093 INFO [Thread-596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-596. 2024-11-07T15:23:27,093 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-11-07T15:23:27,093 DEBUG [Thread-596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:27,093 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:27,093 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:27,093 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:27,093 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:27,093 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:27,093 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:27,093 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T15:23:27,431 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/553ccae1b9cd4af687752fca7a813ea6 2024-11-07T15:23:27,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/cf5acdd558094d93a46a784a4ac4b9c2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/cf5acdd558094d93a46a784a4ac4b9c2 2024-11-07T15:23:27,440 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/cf5acdd558094d93a46a784a4ac4b9c2, entries=150, sequenceid=449, filesize=30.5 K 2024-11-07T15:23:27,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/bb1eaa2cfc59496f829d3c96562549fc as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/bb1eaa2cfc59496f829d3c96562549fc 2024-11-07T15:23:27,447 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/bb1eaa2cfc59496f829d3c96562549fc, entries=150, sequenceid=449, filesize=12.0 K 2024-11-07T15:23:27,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/553ccae1b9cd4af687752fca7a813ea6 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/553ccae1b9cd4af687752fca7a813ea6 2024-11-07T15:23:27,453 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/553ccae1b9cd4af687752fca7a813ea6, entries=150, sequenceid=449, filesize=12.0 K 2024-11-07T15:23:27,454 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=13.42 KB/13740 for 518e68065b228feb1bbc2642f1a6f79d in 559ms, sequenceid=449, compaction requested=true 2024-11-07T15:23:27,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 518e68065b228feb1bbc2642f1a6f79d: 2024-11-07T15:23:27,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:27,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-07T15:23:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-07T15:23:27,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-07T15:23:27,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3290 sec 2024-11-07T15:23:27,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.3350 sec 2024-11-07T15:23:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T15:23:28,271 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:28,864 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-11-07T15:23:28,864 INFO [Thread-604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-604. 2024-11-07T15:23:28,864 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:28,864 DEBUG [Thread-604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:28,864 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:28,864 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:28,864 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:28,864 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:28,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:28,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:30,293 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T15:23:30,802 INFO [Thread-598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-598. 2024-11-07T15:23:30,802 DEBUG [Thread-598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:30,802 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:30,802 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:30,802 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:30,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T15:23:30,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-07T15:23:30,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-07T15:23:30,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-07T15:23:30,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-07T15:23:30,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-07T15:23:30,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T15:23:30,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3979 2024-11-07T15:23:30,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3955 2024-11-07T15:23:30,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T15:23:30,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1693 2024-11-07T15:23:30,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5078 rows 2024-11-07T15:23:30,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1688 2024-11-07T15:23:30,803 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5062 rows 2024-11-07T15:23:30,803 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T15:23:30,803 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:23:30,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:30,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:30,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:30,804 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T15:23:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T15:23:30,807 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993010807"}]},"ts":"1730993010807"} 2024-11-07T15:23:30,809 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T15:23:30,809 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T15:23:30,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:23:30,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=518e68065b228feb1bbc2642f1a6f79d, UNASSIGN}] 2024-11-07T15:23:30,812 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=518e68065b228feb1bbc2642f1a6f79d, UNASSIGN 2024-11-07T15:23:30,812 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=518e68065b228feb1bbc2642f1a6f79d, regionState=CLOSING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:30,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=518e68065b228feb1bbc2642f1a6f79d, UNASSIGN because future has completed 2024-11-07T15:23:30,814 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:23:30,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:23:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T15:23:30,966 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:30,966 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:23:30,966 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 518e68065b228feb1bbc2642f1a6f79d, disabling compactions & flushes 2024-11-07T15:23:30,966 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:30,966 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:30,966 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. after waiting 0 ms 2024-11-07T15:23:30,966 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:30,966 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 518e68065b228feb1bbc2642f1a6f79d 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T15:23:30,972 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107fdb71733d7f74783a5d3103183b08dec_518e68065b228feb1bbc2642f1a6f79d is 50, key is test_row_0/A:col10/1730993010801/Put/seqid=0 2024-11-07T15:23:30,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742065_1241 (size=12454) 2024-11-07T15:23:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T15:23:31,377 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:31,382 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107fdb71733d7f74783a5d3103183b08dec_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fdb71733d7f74783a5d3103183b08dec_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:31,383 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/80c53657736d4eaaa16c9e96ebf86065, store: [table=TestAcidGuarantees family=A region=518e68065b228feb1bbc2642f1a6f79d] 2024-11-07T15:23:31,383 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/80c53657736d4eaaa16c9e96ebf86065 is 175, key is test_row_0/A:col10/1730993010801/Put/seqid=0 2024-11-07T15:23:31,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742066_1242 (size=31255) 2024-11-07T15:23:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T15:23:31,788 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=457, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/80c53657736d4eaaa16c9e96ebf86065 2024-11-07T15:23:31,796 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/ab0d14ccff094177af36559eabbe65c0 is 50, key is test_row_0/B:col10/1730993010801/Put/seqid=0 2024-11-07T15:23:31,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742067_1243 (size=12301) 2024-11-07T15:23:31,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T15:23:32,201 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/ab0d14ccff094177af36559eabbe65c0 2024-11-07T15:23:32,209 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/9d6bb50d8f16478faa217baedbb09681 is 50, key is test_row_0/C:col10/1730993010801/Put/seqid=0 2024-11-07T15:23:32,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742068_1244 (size=12301) 2024-11-07T15:23:32,614 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/9d6bb50d8f16478faa217baedbb09681 2024-11-07T15:23:32,619 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/A/80c53657736d4eaaa16c9e96ebf86065 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/80c53657736d4eaaa16c9e96ebf86065 2024-11-07T15:23:32,622 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/80c53657736d4eaaa16c9e96ebf86065, entries=150, sequenceid=457, filesize=30.5 K 2024-11-07T15:23:32,623 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/B/ab0d14ccff094177af36559eabbe65c0 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/ab0d14ccff094177af36559eabbe65c0 2024-11-07T15:23:32,626 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/ab0d14ccff094177af36559eabbe65c0, entries=150, sequenceid=457, filesize=12.0 K 2024-11-07T15:23:32,627 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/.tmp/C/9d6bb50d8f16478faa217baedbb09681 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/9d6bb50d8f16478faa217baedbb09681 2024-11-07T15:23:32,630 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/9d6bb50d8f16478faa217baedbb09681, entries=150, sequenceid=457, filesize=12.0 K 2024-11-07T15:23:32,631 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 518e68065b228feb1bbc2642f1a6f79d in 1665ms, sequenceid=457, compaction requested=true 2024-11-07T15:23:32,632 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/133ad1657bce4ee9b05c2eda97eb220e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/bfa8a5996d7740fe84526922fc12454f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/1538cc2a937045ee918fe8ffb132914b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/480dc28724df40e8abad12be40f75c23, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/533d2294abc1442eaa3c01416fa64ac0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/02ee465cc03c43a98260e3205ee2cabe, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/36377a37cf89434fa0b8c33274e8d0de, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/d18becaefd344c8b9cc388cdce6fc082, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ff18382923c945509fd089c4dd539b70, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/d7a721decf6f49ef86947b374092cfd6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/f0e872621c544addb0612913a72b4022, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/b153c66000ff45ae892c92ef7c941a04, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/5ea3e99eef004c8b8addd1d2376a49f8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/bf0d0d082a334e06865022b1f3b55f5a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/a66a4e4e9c6b4d2eb872a85e75ce6c80, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/9975d3d0741d4cd3873e5b45c1e78747, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/f1af365f5c8d4536a8563d9cee00b579, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/88f95f04eded43e4bdbad52695d6aedd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/7fccbdd5e7f64d45aeddd79411fb2b57, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/506d9c52c43b47ba8b614e09a7573c11, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/a0c2da7d1efc478395c15fff3f61de75, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/39086b5ef7344dd9b97fa898ddbd6d65, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ae60e671ead24678b0804fbcbf76d175, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/8eda66eac6ac46eaa0f3066c90eba450, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/0c9f42894b6a4151a9b52e283e158a59, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/b024e0d9098b4835ba4736c3c4be493e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/503284085ff246dfa7037ed0673c35f1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/56f759e18ff84a86be233503d7434b27] to archive 2024-11-07T15:23:32,633 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:23:32,634 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/133ad1657bce4ee9b05c2eda97eb220e to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/133ad1657bce4ee9b05c2eda97eb220e 2024-11-07T15:23:32,635 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/bfa8a5996d7740fe84526922fc12454f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/bfa8a5996d7740fe84526922fc12454f 2024-11-07T15:23:32,637 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/1538cc2a937045ee918fe8ffb132914b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/1538cc2a937045ee918fe8ffb132914b 2024-11-07T15:23:32,638 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/480dc28724df40e8abad12be40f75c23 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/480dc28724df40e8abad12be40f75c23 2024-11-07T15:23:32,639 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/533d2294abc1442eaa3c01416fa64ac0 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/533d2294abc1442eaa3c01416fa64ac0 2024-11-07T15:23:32,640 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/02ee465cc03c43a98260e3205ee2cabe to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/02ee465cc03c43a98260e3205ee2cabe 2024-11-07T15:23:32,642 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/36377a37cf89434fa0b8c33274e8d0de to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/36377a37cf89434fa0b8c33274e8d0de 2024-11-07T15:23:32,643 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/d18becaefd344c8b9cc388cdce6fc082 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/d18becaefd344c8b9cc388cdce6fc082 2024-11-07T15:23:32,644 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ff18382923c945509fd089c4dd539b70 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ff18382923c945509fd089c4dd539b70 2024-11-07T15:23:32,645 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/d7a721decf6f49ef86947b374092cfd6 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/d7a721decf6f49ef86947b374092cfd6 2024-11-07T15:23:32,646 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/f0e872621c544addb0612913a72b4022 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/f0e872621c544addb0612913a72b4022 2024-11-07T15:23:32,648 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/b153c66000ff45ae892c92ef7c941a04 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/b153c66000ff45ae892c92ef7c941a04 2024-11-07T15:23:32,649 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/5ea3e99eef004c8b8addd1d2376a49f8 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/5ea3e99eef004c8b8addd1d2376a49f8 2024-11-07T15:23:32,650 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/bf0d0d082a334e06865022b1f3b55f5a to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/bf0d0d082a334e06865022b1f3b55f5a 2024-11-07T15:23:32,651 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/a66a4e4e9c6b4d2eb872a85e75ce6c80 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/a66a4e4e9c6b4d2eb872a85e75ce6c80 2024-11-07T15:23:32,652 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/9975d3d0741d4cd3873e5b45c1e78747 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/9975d3d0741d4cd3873e5b45c1e78747 2024-11-07T15:23:32,653 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/f1af365f5c8d4536a8563d9cee00b579 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/f1af365f5c8d4536a8563d9cee00b579 2024-11-07T15:23:32,653 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/88f95f04eded43e4bdbad52695d6aedd to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/88f95f04eded43e4bdbad52695d6aedd 2024-11-07T15:23:32,654 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/7fccbdd5e7f64d45aeddd79411fb2b57 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/7fccbdd5e7f64d45aeddd79411fb2b57 2024-11-07T15:23:32,655 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/506d9c52c43b47ba8b614e09a7573c11 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/506d9c52c43b47ba8b614e09a7573c11 2024-11-07T15:23:32,656 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/a0c2da7d1efc478395c15fff3f61de75 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/a0c2da7d1efc478395c15fff3f61de75 2024-11-07T15:23:32,657 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/39086b5ef7344dd9b97fa898ddbd6d65 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/39086b5ef7344dd9b97fa898ddbd6d65 2024-11-07T15:23:32,658 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ae60e671ead24678b0804fbcbf76d175 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ae60e671ead24678b0804fbcbf76d175 2024-11-07T15:23:32,659 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/8eda66eac6ac46eaa0f3066c90eba450 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/8eda66eac6ac46eaa0f3066c90eba450 2024-11-07T15:23:32,660 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/0c9f42894b6a4151a9b52e283e158a59 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/0c9f42894b6a4151a9b52e283e158a59 2024-11-07T15:23:32,661 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/b024e0d9098b4835ba4736c3c4be493e to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/b024e0d9098b4835ba4736c3c4be493e 2024-11-07T15:23:32,663 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/503284085ff246dfa7037ed0673c35f1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/503284085ff246dfa7037ed0673c35f1 2024-11-07T15:23:32,664 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/56f759e18ff84a86be233503d7434b27 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/56f759e18ff84a86be233503d7434b27 2024-11-07T15:23:32,665 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/d9c5a66458974763b6889095e73f882f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/d3d78b21fbc44aada807982657450eb4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/b1bab0c8a60840598c735df36954bfa2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/22ace152cbcd4ea2835b7399b3f7fed8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/7658a690d16e43a58ba6f261f7cbbf99, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/e5193d78fca6444cb4dddd53a509d108, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/2ec14b98104e4d1aa06ed619a6359513, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/52cf0ab26ec54cd5ad3e405ba7658058, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/85f40ba022a747879e8ac394c83e86d6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/89c9b34e63104a1eaf253e4dce3663df, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/725e03e4697d40c899b082b40da7eab2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/18eda101d355472ca413ba971e4e27ae, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/552fa2e954b44e49b69d9fe08797554d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/6a48dd5e6b0a43218ee9238afddc40b0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/6f201245cf934af0b11bd6ffe11c18cd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/53ec97f1e3484c88ba735d6ecdcdb0a2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/5ad32bbf6cff4bec8d7a04c6569a14cf, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/706f1808abb14e1ba7edea84cd57dbf5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/b357accf36544931801311152ebc52c4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/647c2a7b10d34a139c1a509f024af601, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/5a14735a7e044a9b9cdc86475416aaf3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/e3df2789ef254a239401fbe2528fece1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/1cbeeae4ac7344c4883537680ece9575, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/0ecc40212a9a4daa8d36ddaf367f91c1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/52c9b38fe243410288ec7b4b91e75915, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/f76797bad7124a72b193383acdc2e2cd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/3270b9c259214b0da41899a7e23bdcd5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/7714af2b38644450b810934c3a9e9f20] to archive 2024-11-07T15:23:32,666 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:23:32,667 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/d9c5a66458974763b6889095e73f882f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/d9c5a66458974763b6889095e73f882f 2024-11-07T15:23:32,669 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/d3d78b21fbc44aada807982657450eb4 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/d3d78b21fbc44aada807982657450eb4 2024-11-07T15:23:32,669 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/b1bab0c8a60840598c735df36954bfa2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/b1bab0c8a60840598c735df36954bfa2 2024-11-07T15:23:32,670 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/22ace152cbcd4ea2835b7399b3f7fed8 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/22ace152cbcd4ea2835b7399b3f7fed8 2024-11-07T15:23:32,672 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/7658a690d16e43a58ba6f261f7cbbf99 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/7658a690d16e43a58ba6f261f7cbbf99 2024-11-07T15:23:32,673 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/e5193d78fca6444cb4dddd53a509d108 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/e5193d78fca6444cb4dddd53a509d108 2024-11-07T15:23:32,674 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/2ec14b98104e4d1aa06ed619a6359513 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/2ec14b98104e4d1aa06ed619a6359513 2024-11-07T15:23:32,675 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/52cf0ab26ec54cd5ad3e405ba7658058 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/52cf0ab26ec54cd5ad3e405ba7658058 2024-11-07T15:23:32,677 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/85f40ba022a747879e8ac394c83e86d6 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/85f40ba022a747879e8ac394c83e86d6 2024-11-07T15:23:32,678 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/89c9b34e63104a1eaf253e4dce3663df to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/89c9b34e63104a1eaf253e4dce3663df 2024-11-07T15:23:32,679 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/725e03e4697d40c899b082b40da7eab2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/725e03e4697d40c899b082b40da7eab2 2024-11-07T15:23:32,680 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/18eda101d355472ca413ba971e4e27ae to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/18eda101d355472ca413ba971e4e27ae 2024-11-07T15:23:32,682 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/552fa2e954b44e49b69d9fe08797554d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/552fa2e954b44e49b69d9fe08797554d 2024-11-07T15:23:32,683 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/6a48dd5e6b0a43218ee9238afddc40b0 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/6a48dd5e6b0a43218ee9238afddc40b0 2024-11-07T15:23:32,684 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/6f201245cf934af0b11bd6ffe11c18cd to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/6f201245cf934af0b11bd6ffe11c18cd 2024-11-07T15:23:32,685 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/53ec97f1e3484c88ba735d6ecdcdb0a2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/53ec97f1e3484c88ba735d6ecdcdb0a2 2024-11-07T15:23:32,686 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/5ad32bbf6cff4bec8d7a04c6569a14cf to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/5ad32bbf6cff4bec8d7a04c6569a14cf 2024-11-07T15:23:32,687 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/706f1808abb14e1ba7edea84cd57dbf5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/706f1808abb14e1ba7edea84cd57dbf5 2024-11-07T15:23:32,688 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/b357accf36544931801311152ebc52c4 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/b357accf36544931801311152ebc52c4 2024-11-07T15:23:32,689 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/647c2a7b10d34a139c1a509f024af601 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/647c2a7b10d34a139c1a509f024af601 2024-11-07T15:23:32,690 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/5a14735a7e044a9b9cdc86475416aaf3 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/5a14735a7e044a9b9cdc86475416aaf3 2024-11-07T15:23:32,691 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/e3df2789ef254a239401fbe2528fece1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/e3df2789ef254a239401fbe2528fece1 2024-11-07T15:23:32,692 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/1cbeeae4ac7344c4883537680ece9575 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/1cbeeae4ac7344c4883537680ece9575 2024-11-07T15:23:32,693 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/0ecc40212a9a4daa8d36ddaf367f91c1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/0ecc40212a9a4daa8d36ddaf367f91c1 2024-11-07T15:23:32,694 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/52c9b38fe243410288ec7b4b91e75915 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/52c9b38fe243410288ec7b4b91e75915 2024-11-07T15:23:32,695 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/f76797bad7124a72b193383acdc2e2cd to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/f76797bad7124a72b193383acdc2e2cd 2024-11-07T15:23:32,696 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/3270b9c259214b0da41899a7e23bdcd5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/3270b9c259214b0da41899a7e23bdcd5 2024-11-07T15:23:32,697 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/7714af2b38644450b810934c3a9e9f20 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/7714af2b38644450b810934c3a9e9f20 2024-11-07T15:23:32,699 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/152a520d914a48248da9b02121803000, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/3b3cc48af1aa4399a85ddae452202e1e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/be7c80a60bc64b00a4e6b8c44dd1a927, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/8d362e14837842f8ba070fe9c94e50fd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/014f7a3e4c494f3497dbf17e8ae899c4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/7a5d707c9eda4e54a8d0712a679f1a31, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/39baea90b7d04ee08152b81d7c1abeda, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/33a5449892164896b60db5c28b53052c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/5749a265dc1941388134236f2666a8cb, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/532965b432a3457396b7bcc75549f894, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/20e0b5a58774472d947b005599285432, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/0e3400c9047b4f5b98c8bfeead29845d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/ef1c8b75d1d34e81ba0a39c618dc4268, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/4a4c5b964e6e4d1586ec05e3010f9905, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/053995d4d8a24d86b70936a56b06e943, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/2087ecad3e1d41aa9082a2a4233a31e0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/f2c522d6d1e34dd99ea9631b6a3a871d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/d1736c6387a7459687b74ee07723aa3b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/1db9819d01d54a549b4e3f4c3babf6dd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/8926e7efec7244b3aefbd457e4788689, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/19a22a7043a04bfbb3fbda25f2f4afc6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/549957c874834fc093dff65171194e9f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/93e1180ec03e4ae997f6e733c816e623, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/6a60d3332d9b49f78449fa7d496fc578, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/b3449353f6f14d38ba014b363609f300, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/e479de42f2a6427aa56326cfac86bc70, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/eee22a1ac78545f0b3e2b48ad844284c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/a0149618311440729b5041590c1f45ac] to archive 2024-11-07T15:23:32,708 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:23:32,710 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/152a520d914a48248da9b02121803000 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/152a520d914a48248da9b02121803000 2024-11-07T15:23:32,712 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/3b3cc48af1aa4399a85ddae452202e1e to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/3b3cc48af1aa4399a85ddae452202e1e 2024-11-07T15:23:32,714 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/be7c80a60bc64b00a4e6b8c44dd1a927 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/be7c80a60bc64b00a4e6b8c44dd1a927 2024-11-07T15:23:32,715 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/8d362e14837842f8ba070fe9c94e50fd to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/8d362e14837842f8ba070fe9c94e50fd 2024-11-07T15:23:32,719 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/014f7a3e4c494f3497dbf17e8ae899c4 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/014f7a3e4c494f3497dbf17e8ae899c4 2024-11-07T15:23:32,720 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/7a5d707c9eda4e54a8d0712a679f1a31 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/7a5d707c9eda4e54a8d0712a679f1a31 2024-11-07T15:23:32,721 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/39baea90b7d04ee08152b81d7c1abeda to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/39baea90b7d04ee08152b81d7c1abeda 2024-11-07T15:23:32,722 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/33a5449892164896b60db5c28b53052c to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/33a5449892164896b60db5c28b53052c 2024-11-07T15:23:32,723 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/5749a265dc1941388134236f2666a8cb to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/5749a265dc1941388134236f2666a8cb 2024-11-07T15:23:32,725 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/532965b432a3457396b7bcc75549f894 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/532965b432a3457396b7bcc75549f894 2024-11-07T15:23:32,726 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/20e0b5a58774472d947b005599285432 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/20e0b5a58774472d947b005599285432 2024-11-07T15:23:32,727 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/0e3400c9047b4f5b98c8bfeead29845d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/0e3400c9047b4f5b98c8bfeead29845d 2024-11-07T15:23:32,728 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/ef1c8b75d1d34e81ba0a39c618dc4268 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/ef1c8b75d1d34e81ba0a39c618dc4268 2024-11-07T15:23:32,729 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/4a4c5b964e6e4d1586ec05e3010f9905 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/4a4c5b964e6e4d1586ec05e3010f9905 2024-11-07T15:23:32,730 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/053995d4d8a24d86b70936a56b06e943 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/053995d4d8a24d86b70936a56b06e943 2024-11-07T15:23:32,731 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/2087ecad3e1d41aa9082a2a4233a31e0 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/2087ecad3e1d41aa9082a2a4233a31e0 2024-11-07T15:23:32,733 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/f2c522d6d1e34dd99ea9631b6a3a871d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/f2c522d6d1e34dd99ea9631b6a3a871d 2024-11-07T15:23:32,734 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/d1736c6387a7459687b74ee07723aa3b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/d1736c6387a7459687b74ee07723aa3b 2024-11-07T15:23:32,735 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/1db9819d01d54a549b4e3f4c3babf6dd to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/1db9819d01d54a549b4e3f4c3babf6dd 2024-11-07T15:23:32,736 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/8926e7efec7244b3aefbd457e4788689 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/8926e7efec7244b3aefbd457e4788689 2024-11-07T15:23:32,737 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/19a22a7043a04bfbb3fbda25f2f4afc6 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/19a22a7043a04bfbb3fbda25f2f4afc6 2024-11-07T15:23:32,738 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/549957c874834fc093dff65171194e9f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/549957c874834fc093dff65171194e9f 2024-11-07T15:23:32,739 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/93e1180ec03e4ae997f6e733c816e623 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/93e1180ec03e4ae997f6e733c816e623 2024-11-07T15:23:32,741 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/6a60d3332d9b49f78449fa7d496fc578 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/6a60d3332d9b49f78449fa7d496fc578 2024-11-07T15:23:32,742 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/b3449353f6f14d38ba014b363609f300 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/b3449353f6f14d38ba014b363609f300 2024-11-07T15:23:32,743 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/e479de42f2a6427aa56326cfac86bc70 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/e479de42f2a6427aa56326cfac86bc70 2024-11-07T15:23:32,744 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/eee22a1ac78545f0b3e2b48ad844284c to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/eee22a1ac78545f0b3e2b48ad844284c 2024-11-07T15:23:32,746 DEBUG [StoreCloser-TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/a0149618311440729b5041590c1f45ac to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/a0149618311440729b5041590c1f45ac 2024-11-07T15:23:32,759 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/recovered.edits/460.seqid, newMaxSeqId=460, maxSeqId=4 2024-11-07T15:23:32,759 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. 2024-11-07T15:23:32,759 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 518e68065b228feb1bbc2642f1a6f79d: Waiting for close lock at 1730993010966Running coprocessor pre-close hooks at 1730993010966Disabling compacts and flushes for region at 1730993010966Disabling writes for close at 1730993010966Obtaining lock to block concurrent updates at 1730993010967 (+1 ms)Preparing flush snapshotting stores in 518e68065b228feb1bbc2642f1a6f79d at 1730993010967Finished memstore snapshotting TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1730993010967Flushing stores of TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d. at 1730993010967Flushing 518e68065b228feb1bbc2642f1a6f79d/A: creating writer at 1730993010967Flushing mob file 518e68065b228feb1bbc2642f1a6f79d/A: appending metadata at 1730993010972 (+5 ms)Flushing mob file 518e68065b228feb1bbc2642f1a6f79d/A: closing flushed file at 1730993010972Flushing 518e68065b228feb1bbc2642f1a6f79d/A: appending metadata at 1730993011383 (+411 ms)Flushing 518e68065b228feb1bbc2642f1a6f79d/A: closing flushed file at 1730993011383Flushing 518e68065b228feb1bbc2642f1a6f79d/B: creating writer at 1730993011791 (+408 ms)Flushing 518e68065b228feb1bbc2642f1a6f79d/B: appending metadata at 1730993011796 (+5 ms)Flushing 518e68065b228feb1bbc2642f1a6f79d/B: closing flushed file at 1730993011796Flushing 518e68065b228feb1bbc2642f1a6f79d/C: creating writer at 1730993012206 (+410 ms)Flushing 518e68065b228feb1bbc2642f1a6f79d/C: appending metadata at 1730993012209 (+3 ms)Flushing 518e68065b228feb1bbc2642f1a6f79d/C: closing flushed file at 1730993012209Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@716f0e13: reopening flushed file at 1730993012618 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a8f6bc8: reopening flushed file at 1730993012622 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38d036f8: reopening flushed file at 1730993012626 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 518e68065b228feb1bbc2642f1a6f79d in 1665ms, sequenceid=457, compaction requested=true at 1730993012631 (+5 ms)Writing region close event to WAL at 1730993012749 (+118 ms)Running coprocessor post-close hooks at 1730993012759 (+10 ms)Closed at 1730993012759 2024-11-07T15:23:32,761 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:32,762 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=518e68065b228feb1bbc2642f1a6f79d, regionState=CLOSED 2024-11-07T15:23:32,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:23:32,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-07T15:23:32,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 518e68065b228feb1bbc2642f1a6f79d, server=7b3e4e8e12ef,33849,1730992951906 in 1.9500 sec 2024-11-07T15:23:32,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-07T15:23:32,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=518e68065b228feb1bbc2642f1a6f79d, UNASSIGN in 1.9540 sec 2024-11-07T15:23:32,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-07T15:23:32,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9570 sec 2024-11-07T15:23:32,769 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993012769"}]},"ts":"1730993012769"} 2024-11-07T15:23:32,771 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T15:23:32,771 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T15:23:32,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9670 sec 2024-11-07T15:23:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T15:23:32,951 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:32,952 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T15:23:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:32,954 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-07T15:23:32,954 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:32,957 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:32,961 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/recovered.edits] 2024-11-07T15:23:32,965 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/34c41dbae860485298dd0b7cedfb0402 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/34c41dbae860485298dd0b7cedfb0402 2024-11-07T15:23:32,966 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/80c53657736d4eaaa16c9e96ebf86065 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/80c53657736d4eaaa16c9e96ebf86065 2024-11-07T15:23:32,968 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ab6dd47c1d5f452ba011b7065d111001 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/ab6dd47c1d5f452ba011b7065d111001 2024-11-07T15:23:32,969 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/cf5acdd558094d93a46a784a4ac4b9c2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/A/cf5acdd558094d93a46a784a4ac4b9c2 2024-11-07T15:23:32,971 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/6eaba82e3fd942b6bb31f74d2fd36d37 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/6eaba82e3fd942b6bb31f74d2fd36d37 2024-11-07T15:23:32,972 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/ab0d14ccff094177af36559eabbe65c0 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/ab0d14ccff094177af36559eabbe65c0 2024-11-07T15:23:32,974 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/bac9f77ba9e444659fcdfb7fecd7a05f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/bac9f77ba9e444659fcdfb7fecd7a05f 2024-11-07T15:23:32,975 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/bb1eaa2cfc59496f829d3c96562549fc to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/B/bb1eaa2cfc59496f829d3c96562549fc 2024-11-07T15:23:32,977 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/3ec92f4ea34b4e778c860a87638ba479 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/3ec92f4ea34b4e778c860a87638ba479 2024-11-07T15:23:32,978 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/553ccae1b9cd4af687752fca7a813ea6 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/553ccae1b9cd4af687752fca7a813ea6 2024-11-07T15:23:32,979 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/9d6bb50d8f16478faa217baedbb09681 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/9d6bb50d8f16478faa217baedbb09681 2024-11-07T15:23:32,980 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/e3ff8f2f75bf4d93b444670819a60e55 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/C/e3ff8f2f75bf4d93b444670819a60e55 2024-11-07T15:23:32,982 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/recovered.edits/460.seqid to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d/recovered.edits/460.seqid 2024-11-07T15:23:32,983 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:32,983 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T15:23:32,983 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T15:23:32,984 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T15:23:32,988 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107087a03b96e7d484490a33a9541fe5024_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107087a03b96e7d484490a33a9541fe5024_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:32,989 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110709eca3a6961246f19c4587c4598bcf39_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110709eca3a6961246f19c4587c4598bcf39_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:32,990 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110712e818d7c87b4160929fe707e52e50e7_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110712e818d7c87b4160929fe707e52e50e7_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:32,991 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071fa6e87567414a2ea1027e159f9b246b_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071fa6e87567414a2ea1027e159f9b246b_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:32,992 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110722576fd8919b4c93aec680547d5d8f90_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110722576fd8919b4c93aec680547d5d8f90_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:32,993 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072ce2b9431dea4a43a125afba0cc41133_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072ce2b9431dea4a43a125afba0cc41133_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:32,994 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072dcef0dd0d2044acb5c0540686b23f7a_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072dcef0dd0d2044acb5c0540686b23f7a_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:32,995 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074171f4df299448bab92051038a4f433a_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074171f4df299448bab92051038a4f433a_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:32,996 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107459207d748154b32b3aa13cb0cf40df6_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107459207d748154b32b3aa13cb0cf40df6_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:32,997 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074ca1d206ef0e47dc9c0627c0d4060281_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074ca1d206ef0e47dc9c0627c0d4060281_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:32,998 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075d5de20648334cdf988af05860fc2d3e_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075d5de20648334cdf988af05860fc2d3e_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:32,999 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076154f8fd202d4e929e9d77b0758eaf47_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076154f8fd202d4e929e9d77b0758eaf47_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:33,000 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076bb5d25879a746ccaeec5d871e706bfc_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076bb5d25879a746ccaeec5d871e706bfc_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:33,001 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077b98b682e5ca4a24907e4b019b9a5c1b_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077b98b682e5ca4a24907e4b019b9a5c1b_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:33,002 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077de6db62ce8c4a1581acc499ad78a226_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411077de6db62ce8c4a1581acc499ad78a226_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:33,003 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b4147144ded6449c99b0e3f7e9d72c10_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b4147144ded6449c99b0e3f7e9d72c10_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:33,004 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c2b96571908f41a981df9538d30dc313_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c2b96571908f41a981df9538d30dc313_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:33,005 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c6308084d7f3489b82d2049df09af4e7_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c6308084d7f3489b82d2049df09af4e7_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:33,005 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cb620feccdb842709f988038de1e868a_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cb620feccdb842709f988038de1e868a_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:33,006 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cd086bf149a042578a92d0f878072e8a_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cd086bf149a042578a92d0f878072e8a_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:33,007 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f043b94e0b0343769f9057e08c36afd7_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f043b94e0b0343769f9057e08c36afd7_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:33,008 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f208c350c35845dcb8334132c0477b94_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f208c350c35845dcb8334132c0477b94_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:33,009 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fdb71733d7f74783a5d3103183b08dec_518e68065b228feb1bbc2642f1a6f79d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fdb71733d7f74783a5d3103183b08dec_518e68065b228feb1bbc2642f1a6f79d 2024-11-07T15:23:33,009 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T15:23:33,011 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:33,013 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T15:23:33,015 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T15:23:33,016 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:33,016 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T15:23:33,016 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730993013016"}]},"ts":"9223372036854775807"} 2024-11-07T15:23:33,018 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T15:23:33,018 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 518e68065b228feb1bbc2642f1a6f79d, NAME => 'TestAcidGuarantees,,1730992983828.518e68065b228feb1bbc2642f1a6f79d.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T15:23:33,018 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T15:23:33,018 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730993013018"}]},"ts":"9223372036854775807"} 2024-11-07T15:23:33,020 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T15:23:33,023 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:33,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 71 msec 2024-11-07T15:23:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-07T15:23:33,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T15:23:33,061 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:33,072 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239 (was 240), OpenFileDescriptor=457 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=465 (was 474), ProcessCount=11 (was 11), AvailableMemoryMB=1429 (was 1635) 2024-11-07T15:23:33,080 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=465, ProcessCount=11, AvailableMemoryMB=1428 2024-11-07T15:23:33,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:23:33,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:23:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:33,083 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T15:23:33,084 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:33,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-07T15:23:33,084 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T15:23:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T15:23:33,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742069_1245 (size=959) 2024-11-07T15:23:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T15:23:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T15:23:33,496 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938 2024-11-07T15:23:33,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742070_1246 (size=53) 2024-11-07T15:23:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T15:23:33,904 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:23:33,904 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b93a997d5d499fc85333e4dccefd8493, disabling compactions & flushes 2024-11-07T15:23:33,904 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:33,904 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:33,904 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. after waiting 0 ms 2024-11-07T15:23:33,904 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:33,904 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:33,904 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b93a997d5d499fc85333e4dccefd8493: Waiting for close lock at 1730993013904Disabling compacts and flushes for region at 1730993013904Disabling writes for close at 1730993013904Writing region close event to WAL at 1730993013904Closed at 1730993013904 2024-11-07T15:23:33,906 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T15:23:33,906 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730993013906"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730993013906"}]},"ts":"1730993013906"} 2024-11-07T15:23:33,907 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T15:23:33,908 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T15:23:33,909 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993013908"}]},"ts":"1730993013908"} 2024-11-07T15:23:33,910 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T15:23:33,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b93a997d5d499fc85333e4dccefd8493, ASSIGN}] 2024-11-07T15:23:33,912 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b93a997d5d499fc85333e4dccefd8493, ASSIGN 2024-11-07T15:23:33,912 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b93a997d5d499fc85333e4dccefd8493, ASSIGN; state=OFFLINE, location=7b3e4e8e12ef,33849,1730992951906; forceNewPlan=false, retain=false 2024-11-07T15:23:34,063 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=b93a997d5d499fc85333e4dccefd8493, regionState=OPENING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:34,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b93a997d5d499fc85333e4dccefd8493, ASSIGN because future has completed 2024-11-07T15:23:34,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:23:34,220 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:34,220 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:23:34,220 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:34,220 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:23:34,221 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:34,221 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:34,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T15:23:34,222 INFO [StoreOpener-b93a997d5d499fc85333e4dccefd8493-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:34,224 INFO [StoreOpener-b93a997d5d499fc85333e4dccefd8493-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b93a997d5d499fc85333e4dccefd8493 columnFamilyName A 2024-11-07T15:23:34,224 DEBUG [StoreOpener-b93a997d5d499fc85333e4dccefd8493-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:34,224 INFO [StoreOpener-b93a997d5d499fc85333e4dccefd8493-1 {}] regionserver.HStore(327): Store=b93a997d5d499fc85333e4dccefd8493/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:23:34,224 INFO [StoreOpener-b93a997d5d499fc85333e4dccefd8493-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:34,225 INFO [StoreOpener-b93a997d5d499fc85333e4dccefd8493-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b93a997d5d499fc85333e4dccefd8493 columnFamilyName B 2024-11-07T15:23:34,225 DEBUG [StoreOpener-b93a997d5d499fc85333e4dccefd8493-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:34,226 INFO [StoreOpener-b93a997d5d499fc85333e4dccefd8493-1 {}] regionserver.HStore(327): Store=b93a997d5d499fc85333e4dccefd8493/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:23:34,226 INFO [StoreOpener-b93a997d5d499fc85333e4dccefd8493-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:34,227 INFO [StoreOpener-b93a997d5d499fc85333e4dccefd8493-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b93a997d5d499fc85333e4dccefd8493 columnFamilyName C 2024-11-07T15:23:34,227 DEBUG [StoreOpener-b93a997d5d499fc85333e4dccefd8493-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:23:34,227 INFO [StoreOpener-b93a997d5d499fc85333e4dccefd8493-1 {}] regionserver.HStore(327): Store=b93a997d5d499fc85333e4dccefd8493/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:23:34,228 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:34,228 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:34,228 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:34,229 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:34,229 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:34,230 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:23:34,232 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:34,234 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T15:23:34,234 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened b93a997d5d499fc85333e4dccefd8493; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66869059, jitterRate=-0.003573372960090637}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:23:34,234 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:34,235 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for b93a997d5d499fc85333e4dccefd8493: Running coprocessor pre-open hook at 1730993014221Writing region info on filesystem at 1730993014221Initializing all the Stores at 1730993014222 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993014222Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993014222Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993014222Cleaning up temporary data from old regions at 1730993014230 (+8 ms)Running coprocessor post-open hooks at 1730993014234 (+4 ms)Region opened successfully at 1730993014235 (+1 ms) 2024-11-07T15:23:34,235 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., pid=63, masterSystemTime=1730993014217 2024-11-07T15:23:34,237 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:34,237 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:34,238 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=b93a997d5d499fc85333e4dccefd8493, regionState=OPEN, openSeqNum=2, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:34,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:23:34,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-07T15:23:34,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 in 175 msec 2024-11-07T15:23:34,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-07T15:23:34,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b93a997d5d499fc85333e4dccefd8493, ASSIGN in 332 msec 2024-11-07T15:23:34,244 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T15:23:34,244 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993014244"}]},"ts":"1730993014244"} 2024-11-07T15:23:34,246 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T15:23:34,247 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T15:23:34,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-07T15:23:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-07T15:23:35,232 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:35,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6f1434, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,233 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:35,233 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:35,234 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:35,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9830fb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:35,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:35,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:35,236 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:35,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62260cf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,237 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:35,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:35,239 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T15:23:35,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b3e4e8e12ef,46811,1730992951165 2024-11-07T15:23:35,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@611da558 2024-11-07T15:23:35,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T15:23:35,241 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T15:23:35,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec2896e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,243 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:35,243 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:35,244 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38b9fed9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:35,246 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a0f56c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5155197b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,247 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:35,247 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:35,248 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79853070, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:35,251 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44e45c75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0e849b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,252 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:35,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:35,253 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d83aa0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:35,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:35,255 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:35,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bfeec4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0cbf10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,257 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:35,257 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:35,258 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:35,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:35,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:35,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cd9917c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:35,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:35,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:35,260 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:35,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@605efb2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5735c732, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,261 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:35,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:35,262 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:35,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:35,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b6ecdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:35,264 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:35,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e34c43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6493a71c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,267 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:35,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:35,268 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:35,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:35,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:35,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@784d84cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:35,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:35,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:35,269 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72bfbebd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a1b1c7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,271 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:35,271 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:35,272 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44551cca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:35,274 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e1f2306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@727224d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,276 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:35,276 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:35,277 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53bdf5a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:35,281 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:35,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e4ca728, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279c118e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:35,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:35,283 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:35,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22bb83ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:35,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:35,285 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b6216, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@501877a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:23:35,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:23:35,287 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:23:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:23:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:23:35,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511df334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:23:35,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:23:35,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:35,289 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:23:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@567bad54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:23:35,295 DEBUG [Thread-1145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:35,296 DEBUG [Thread-1156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:35,298 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:35,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:23:35,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:23:35,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-07T15:23:35,310 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T15:23:35,311 DEBUG [Thread-1162 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:35,312 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:35,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:35,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:23:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:35,325 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:35,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:23:35,328 DEBUG [Thread-1151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:35,332 DEBUG [Thread-1154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:35,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:35,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:23:35,335 DEBUG [Thread-1154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:35,336 DEBUG [Thread-1154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:35,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:23:35,344 DEBUG [Thread-1160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:35,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:23:35,358 DEBUG [Thread-1143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:35,359 DEBUG [Thread-1143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:35,359 DEBUG [Thread-1143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:35,361 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:23:35,363 DEBUG [Thread-1158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:35,364 DEBUG [Thread-1158 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:35,369 DEBUG [Thread-1158 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:35,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:35,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/a90e47b3e42a453e8462a11ef1abb11e is 50, key is test_row_0/A:col10/1730993015315/Put/seqid=0 2024-11-07T15:23:35,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:23:35,376 DEBUG [Thread-1149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:35,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:35,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:35,379 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:35,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:23:35,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993075385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:35,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993075387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:35,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993075388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:35,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993075390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:35,400 DEBUG [Thread-1147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:23:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:23:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:23:35,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:23:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:23:35,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993075409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:35,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T15:23:35,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742071_1247 (size=12001) 2024-11-07T15:23:35,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T15:23:35,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:35,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:35,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:35,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:35,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:35,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:35,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993075501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:35,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993075501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:35,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993075501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:35,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993075501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:35,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993075521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T15:23:35,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T15:23:35,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:35,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:35,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:35,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:35,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:35,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993075721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:35,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993075721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:35,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993075721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:35,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993075721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:35,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993075734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:35,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:35,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T15:23:35,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:35,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:35,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:35,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:35,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:35,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/a90e47b3e42a453e8462a11ef1abb11e 2024-11-07T15:23:35,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/706e7d1b408a483aaf944888c0eb019a is 50, key is test_row_0/B:col10/1730993015315/Put/seqid=0 2024-11-07T15:23:35,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742072_1248 (size=12001) 2024-11-07T15:23:35,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/706e7d1b408a483aaf944888c0eb019a 2024-11-07T15:23:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T15:23:35,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T15:23:35,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:35,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:35,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:35,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:35,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:35,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/279c8e02a2144ff89820ae11ff3eb8da is 50, key is test_row_0/C:col10/1730993015315/Put/seqid=0 2024-11-07T15:23:35,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742073_1249 (size=12001) 2024-11-07T15:23:36,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993076031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993076031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993076031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993076031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993076040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T15:23:36,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:36,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:36,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:36,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T15:23:36,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:36,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:36,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:36,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/279c8e02a2144ff89820ae11ff3eb8da 2024-11-07T15:23:36,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/a90e47b3e42a453e8462a11ef1abb11e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/a90e47b3e42a453e8462a11ef1abb11e 2024-11-07T15:23:36,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/a90e47b3e42a453e8462a11ef1abb11e, entries=150, sequenceid=12, filesize=11.7 K 2024-11-07T15:23:36,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/706e7d1b408a483aaf944888c0eb019a as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/706e7d1b408a483aaf944888c0eb019a 2024-11-07T15:23:36,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/706e7d1b408a483aaf944888c0eb019a, entries=150, sequenceid=12, filesize=11.7 K 2024-11-07T15:23:36,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/279c8e02a2144ff89820ae11ff3eb8da as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/279c8e02a2144ff89820ae11ff3eb8da 2024-11-07T15:23:36,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T15:23:36,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:36,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:36,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:36,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/279c8e02a2144ff89820ae11ff3eb8da, entries=150, sequenceid=12, filesize=11.7 K 2024-11-07T15:23:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for b93a997d5d499fc85333e4dccefd8493 in 1085ms, sequenceid=12, compaction requested=false 2024-11-07T15:23:36,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T15:23:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:36,543 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:36,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T15:23:36,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/1a7e62004b404d2ea2a70398cedfa260 is 50, key is test_row_0/A:col10/1730993015371/Put/seqid=0 2024-11-07T15:23:36,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T15:23:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:36,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993076555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742074_1250 (size=12001) 2024-11-07T15:23:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993076556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993076556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/1a7e62004b404d2ea2a70398cedfa260 2024-11-07T15:23:36,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993076565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993076569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/f39d30afb1ca4d01b3f08c6fc20d0b17 is 50, key is test_row_0/B:col10/1730993015371/Put/seqid=0 2024-11-07T15:23:36,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742075_1251 (size=12001) 2024-11-07T15:23:36,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/f39d30afb1ca4d01b3f08c6fc20d0b17 2024-11-07T15:23:36,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/7c59da3bb53044359076398a223f89ad is 50, key is test_row_0/C:col10/1730993015371/Put/seqid=0 2024-11-07T15:23:36,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742076_1252 (size=12001) 2024-11-07T15:23:36,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/7c59da3bb53044359076398a223f89ad 2024-11-07T15:23:36,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/1a7e62004b404d2ea2a70398cedfa260 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/1a7e62004b404d2ea2a70398cedfa260 2024-11-07T15:23:36,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/1a7e62004b404d2ea2a70398cedfa260, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T15:23:36,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/f39d30afb1ca4d01b3f08c6fc20d0b17 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/f39d30afb1ca4d01b3f08c6fc20d0b17 2024-11-07T15:23:36,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/f39d30afb1ca4d01b3f08c6fc20d0b17, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T15:23:36,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/7c59da3bb53044359076398a223f89ad as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/7c59da3bb53044359076398a223f89ad 2024-11-07T15:23:36,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/7c59da3bb53044359076398a223f89ad, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T15:23:36,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for b93a997d5d499fc85333e4dccefd8493 in 133ms, sequenceid=38, compaction requested=false 2024-11-07T15:23:36,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:36,677 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:36,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T15:23:36,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/3f543971b5674376bfabfd66500a56d8 is 50, key is test_row_0/A:col10/1730993016554/Put/seqid=0 2024-11-07T15:23:36,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742077_1253 (size=14341) 2024-11-07T15:23:36,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T15:23:36,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:36,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:36,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:36,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993076714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993076714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993076722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993076727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993076727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993076831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993076831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993076831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993076840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993076840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:36,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:36,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:36,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T15:23:36,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:36,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:36,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:36,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:37,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T15:23:37,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:37,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:37,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:37,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:37,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:37,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:37,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:37,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993077041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:37,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993077041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:37,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993077041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:37,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993077050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:37,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:37,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993077051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:37,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:37,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:37,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/3f543971b5674376bfabfd66500a56d8 2024-11-07T15:23:37,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/9c9caec1aa544459b654b892313db501 is 50, key is test_row_0/B:col10/1730993016554/Put/seqid=0 2024-11-07T15:23:37,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742078_1254 (size=12001) 2024-11-07T15:23:37,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/9c9caec1aa544459b654b892313db501 2024-11-07T15:23:37,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/3e808f1410cc4ad48d9d2d24d144a8cb is 50, key is test_row_0/C:col10/1730993016554/Put/seqid=0 2024-11-07T15:23:37,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T15:23:37,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:37,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:37,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:37,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:37,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:37,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742079_1255 (size=12001) 2024-11-07T15:23:37,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T15:23:37,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:37,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:37,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:37,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:37,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:37,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993077350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:37,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:37,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993077350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:37,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993077350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:37,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993077361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:37,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:37,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993077361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:37,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T15:23:37,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T15:23:37,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:37,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:37,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:37,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:37,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:37,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/3e808f1410cc4ad48d9d2d24d144a8cb 2024-11-07T15:23:37,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/3f543971b5674376bfabfd66500a56d8 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/3f543971b5674376bfabfd66500a56d8 2024-11-07T15:23:37,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/3f543971b5674376bfabfd66500a56d8, entries=200, sequenceid=53, filesize=14.0 K 2024-11-07T15:23:37,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/9c9caec1aa544459b654b892313db501 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/9c9caec1aa544459b654b892313db501 2024-11-07T15:23:37,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/9c9caec1aa544459b654b892313db501, entries=150, sequenceid=53, filesize=11.7 K 2024-11-07T15:23:37,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/3e808f1410cc4ad48d9d2d24d144a8cb as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/3e808f1410cc4ad48d9d2d24d144a8cb 2024-11-07T15:23:37,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/3e808f1410cc4ad48d9d2d24d144a8cb, entries=150, sequenceid=53, filesize=11.7 K 2024-11-07T15:23:37,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for b93a997d5d499fc85333e4dccefd8493 in 979ms, sequenceid=53, compaction requested=true 2024-11-07T15:23:37,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:37,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:37,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:37,656 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:37,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:37,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:37,656 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:37,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:37,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:37,658 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:37,658 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:37,658 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/B is initiating minor compaction (all files) 2024-11-07T15:23:37,658 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/A is initiating minor compaction (all files) 2024-11-07T15:23:37,658 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/A in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:37,658 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/a90e47b3e42a453e8462a11ef1abb11e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/1a7e62004b404d2ea2a70398cedfa260, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/3f543971b5674376bfabfd66500a56d8] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=37.4 K 2024-11-07T15:23:37,659 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/B in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:37,659 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/706e7d1b408a483aaf944888c0eb019a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/f39d30afb1ca4d01b3f08c6fc20d0b17, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/9c9caec1aa544459b654b892313db501] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=35.2 K 2024-11-07T15:23:37,659 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting a90e47b3e42a453e8462a11ef1abb11e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1730993015312 2024-11-07T15:23:37,659 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 706e7d1b408a483aaf944888c0eb019a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1730993015312 2024-11-07T15:23:37,659 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a7e62004b404d2ea2a70398cedfa260, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730993015371 2024-11-07T15:23:37,660 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting f39d30afb1ca4d01b3f08c6fc20d0b17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730993015371 2024-11-07T15:23:37,660 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f543971b5674376bfabfd66500a56d8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730993016554 2024-11-07T15:23:37,660 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c9caec1aa544459b654b892313db501, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730993016554 2024-11-07T15:23:37,670 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#B#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:37,670 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/097c7cdae77d4e7c850f5fcd6a1f08ba is 50, key is test_row_0/B:col10/1730993016554/Put/seqid=0 2024-11-07T15:23:37,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-07T15:23:37,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:37,675 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T15:23:37,678 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#A#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:37,678 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/6adb6acd7df648d3a765e2af882fcab6 is 50, key is test_row_0/A:col10/1730993016554/Put/seqid=0 2024-11-07T15:23:37,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742080_1256 (size=12104) 2024-11-07T15:23:37,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/48c6e8de8a0c4b02ad12773c8e3f4eb1 is 50, key is test_row_0/A:col10/1730993016719/Put/seqid=0 2024-11-07T15:23:37,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742082_1258 (size=12001) 2024-11-07T15:23:37,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742081_1257 (size=12104) 2024-11-07T15:23:37,736 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/6adb6acd7df648d3a765e2af882fcab6 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/6adb6acd7df648d3a765e2af882fcab6 2024-11-07T15:23:37,756 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/A of b93a997d5d499fc85333e4dccefd8493 into 6adb6acd7df648d3a765e2af882fcab6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:37,756 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:37,756 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/A, priority=13, startTime=1730993017656; duration=0sec 2024-11-07T15:23:37,757 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:37,758 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:A 2024-11-07T15:23:37,758 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:37,760 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:37,760 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/C is initiating minor compaction (all files) 2024-11-07T15:23:37,760 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/C in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:37,760 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/279c8e02a2144ff89820ae11ff3eb8da, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/7c59da3bb53044359076398a223f89ad, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/3e808f1410cc4ad48d9d2d24d144a8cb] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=35.2 K 2024-11-07T15:23:37,779 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 279c8e02a2144ff89820ae11ff3eb8da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1730993015312 2024-11-07T15:23:37,779 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c59da3bb53044359076398a223f89ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730993015371 2024-11-07T15:23:37,780 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e808f1410cc4ad48d9d2d24d144a8cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730993016554 2024-11-07T15:23:37,809 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#C#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:37,810 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/c42044e081d24b098eab7400cd630a38 is 50, key is test_row_0/C:col10/1730993016554/Put/seqid=0 2024-11-07T15:23:37,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742083_1259 (size=12104) 2024-11-07T15:23:37,846 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/c42044e081d24b098eab7400cd630a38 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/c42044e081d24b098eab7400cd630a38 2024-11-07T15:23:37,853 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/C of b93a997d5d499fc85333e4dccefd8493 into c42044e081d24b098eab7400cd630a38(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:37,853 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:37,853 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/C, priority=13, startTime=1730993017656; duration=0sec 2024-11-07T15:23:37,853 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:37,853 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:C 2024-11-07T15:23:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:37,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:37,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993077887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:37,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:37,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993077888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993077887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:37,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993077891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:37,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993077894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:38,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:38,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993078000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993078001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:38,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:38,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993078001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993078001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:38,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993078001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:38,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:38,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:38,096 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/097c7cdae77d4e7c850f5fcd6a1f08ba as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/097c7cdae77d4e7c850f5fcd6a1f08ba 2024-11-07T15:23:38,101 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/B of b93a997d5d499fc85333e4dccefd8493 into 097c7cdae77d4e7c850f5fcd6a1f08ba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:38,101 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:38,101 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/B, priority=13, startTime=1730993017656; duration=0sec 2024-11-07T15:23:38,101 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:38,101 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:B 2024-11-07T15:23:38,105 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/48c6e8de8a0c4b02ad12773c8e3f4eb1 2024-11-07T15:23:38,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/e4d6ed71e5394db39067bcf5018abc46 is 50, key is test_row_0/B:col10/1730993016719/Put/seqid=0 2024-11-07T15:23:38,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742084_1260 (size=12001) 2024-11-07T15:23:38,144 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/e4d6ed71e5394db39067bcf5018abc46 2024-11-07T15:23:38,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/98c2e12a72c04862b6da9451063b5b4a is 50, key is test_row_0/C:col10/1730993016719/Put/seqid=0 2024-11-07T15:23:38,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742085_1261 (size=12001) 2024-11-07T15:23:38,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:38,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993078211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:38,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:38,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:38,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993078211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:38,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:38,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993078211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:38,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993078211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:38,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993078211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:38,289 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T15:23:38,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993078521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:38,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:38,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993078524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993078524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:38,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993078524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993078525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:38,562 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/98c2e12a72c04862b6da9451063b5b4a 2024-11-07T15:23:38,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/48c6e8de8a0c4b02ad12773c8e3f4eb1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/48c6e8de8a0c4b02ad12773c8e3f4eb1 2024-11-07T15:23:38,578 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/48c6e8de8a0c4b02ad12773c8e3f4eb1, entries=150, sequenceid=74, filesize=11.7 K 2024-11-07T15:23:38,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/e4d6ed71e5394db39067bcf5018abc46 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/e4d6ed71e5394db39067bcf5018abc46 2024-11-07T15:23:38,582 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/e4d6ed71e5394db39067bcf5018abc46, entries=150, sequenceid=74, filesize=11.7 K 2024-11-07T15:23:38,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/98c2e12a72c04862b6da9451063b5b4a as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/98c2e12a72c04862b6da9451063b5b4a 2024-11-07T15:23:38,587 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/98c2e12a72c04862b6da9451063b5b4a, entries=150, sequenceid=74, filesize=11.7 K 2024-11-07T15:23:38,587 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for b93a997d5d499fc85333e4dccefd8493 in 913ms, sequenceid=74, compaction requested=false 2024-11-07T15:23:38,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:38,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:38,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-07T15:23:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-07T15:23:38,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-07T15:23:38,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2770 sec 2024-11-07T15:23:38,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 3.2830 sec 2024-11-07T15:23:39,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:39,034 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:39,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T15:23:39,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/a2519b2cf64d4655bbd53422749540c0 is 50, key is test_row_0/A:col10/1730993017879/Put/seqid=0 2024-11-07T15:23:39,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742086_1262 (size=14341) 2024-11-07T15:23:39,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993079055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993079057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993079057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993079058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993079059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993079171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993079171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993079171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993079171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993079171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993079381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993079383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993079384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993079384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993079384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/a2519b2cf64d4655bbd53422749540c0 2024-11-07T15:23:39,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/d4676a05163d4c04a4961d579f8fa59a is 50, key is test_row_0/B:col10/1730993017879/Put/seqid=0 2024-11-07T15:23:39,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742087_1263 (size=12001) 2024-11-07T15:23:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-07T15:23:39,471 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:39,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-07T15:23:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T15:23:39,476 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:39,477 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:39,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T15:23:39,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T15:23:39,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:39,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:39,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:39,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:39,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:39,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993079691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993079691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993079691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993079704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:39,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993079704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:39,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:39,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T15:23:39,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T15:23:39,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:39,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:39,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:39,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:39,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:39,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/d4676a05163d4c04a4961d579f8fa59a 2024-11-07T15:23:39,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/5f7f4b19021f4a7ab5f7b4d830a91e1b is 50, key is test_row_0/C:col10/1730993017879/Put/seqid=0 2024-11-07T15:23:39,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742088_1264 (size=12001) 2024-11-07T15:23:39,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/5f7f4b19021f4a7ab5f7b4d830a91e1b 2024-11-07T15:23:39,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/a2519b2cf64d4655bbd53422749540c0 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/a2519b2cf64d4655bbd53422749540c0 2024-11-07T15:23:39,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/a2519b2cf64d4655bbd53422749540c0, entries=200, sequenceid=94, filesize=14.0 K 2024-11-07T15:23:39,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/d4676a05163d4c04a4961d579f8fa59a as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/d4676a05163d4c04a4961d579f8fa59a 2024-11-07T15:23:39,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/d4676a05163d4c04a4961d579f8fa59a, entries=150, sequenceid=94, filesize=11.7 K 2024-11-07T15:23:39,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/5f7f4b19021f4a7ab5f7b4d830a91e1b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/5f7f4b19021f4a7ab5f7b4d830a91e1b 2024-11-07T15:23:39,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/5f7f4b19021f4a7ab5f7b4d830a91e1b, entries=150, sequenceid=94, filesize=11.7 K 2024-11-07T15:23:39,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for b93a997d5d499fc85333e4dccefd8493 in 882ms, sequenceid=94, compaction requested=true 2024-11-07T15:23:39,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:39,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:39,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:39,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:39,917 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:39,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:39,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:39,917 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:39,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:39,918 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:39,918 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/B is initiating minor compaction (all files) 2024-11-07T15:23:39,918 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/B in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:39,918 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/097c7cdae77d4e7c850f5fcd6a1f08ba, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/e4d6ed71e5394db39067bcf5018abc46, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/d4676a05163d4c04a4961d579f8fa59a] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=35.3 K 2024-11-07T15:23:39,918 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:39,918 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/A is initiating minor compaction (all files) 2024-11-07T15:23:39,918 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/A in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:39,918 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/6adb6acd7df648d3a765e2af882fcab6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/48c6e8de8a0c4b02ad12773c8e3f4eb1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/a2519b2cf64d4655bbd53422749540c0] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=37.5 K 2024-11-07T15:23:39,918 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 097c7cdae77d4e7c850f5fcd6a1f08ba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730993016554 2024-11-07T15:23:39,919 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6adb6acd7df648d3a765e2af882fcab6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730993016554 2024-11-07T15:23:39,919 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting e4d6ed71e5394db39067bcf5018abc46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1730993016710 2024-11-07T15:23:39,919 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48c6e8de8a0c4b02ad12773c8e3f4eb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1730993016710 2024-11-07T15:23:39,919 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting d4676a05163d4c04a4961d579f8fa59a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730993017879 2024-11-07T15:23:39,920 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2519b2cf64d4655bbd53422749540c0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730993017871 2024-11-07T15:23:39,946 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#B#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:39,947 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/7954d864022443c3ae14ead53a57579d is 50, key is test_row_0/B:col10/1730993017879/Put/seqid=0 2024-11-07T15:23:39,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T15:23:39,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:39,950 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T15:23:39,966 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#A#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:39,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/5dd6bb70a080496886f382fff38ed63b is 50, key is test_row_0/A:col10/1730993019055/Put/seqid=0 2024-11-07T15:23:39,966 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/afb4ad3123f3479a806e3088886f0c47 is 50, key is test_row_0/A:col10/1730993017879/Put/seqid=0 2024-11-07T15:23:39,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742089_1265 (size=12207) 2024-11-07T15:23:39,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742091_1267 (size=12001) 2024-11-07T15:23:39,999 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/5dd6bb70a080496886f382fff38ed63b 2024-11-07T15:23:40,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742090_1266 (size=12207) 2024-11-07T15:23:40,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/16263a8c967d449fbacb68d195621124 is 50, key is test_row_0/B:col10/1730993019055/Put/seqid=0 2024-11-07T15:23:40,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742092_1268 (size=12001) 2024-11-07T15:23:40,038 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/16263a8c967d449fbacb68d195621124 2024-11-07T15:23:40,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/10e329d0b6414e49b7af99af272ff504 is 50, key is test_row_0/C:col10/1730993019055/Put/seqid=0 2024-11-07T15:23:40,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742093_1269 (size=12001) 2024-11-07T15:23:40,081 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/10e329d0b6414e49b7af99af272ff504 2024-11-07T15:23:40,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/5dd6bb70a080496886f382fff38ed63b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/5dd6bb70a080496886f382fff38ed63b 2024-11-07T15:23:40,089 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/5dd6bb70a080496886f382fff38ed63b, entries=150, sequenceid=113, filesize=11.7 K 2024-11-07T15:23:40,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/16263a8c967d449fbacb68d195621124 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/16263a8c967d449fbacb68d195621124 2024-11-07T15:23:40,096 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/16263a8c967d449fbacb68d195621124, entries=150, sequenceid=113, filesize=11.7 K 2024-11-07T15:23:40,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/10e329d0b6414e49b7af99af272ff504 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/10e329d0b6414e49b7af99af272ff504 2024-11-07T15:23:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T15:23:40,108 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/10e329d0b6414e49b7af99af272ff504, entries=150, sequenceid=113, filesize=11.7 K 2024-11-07T15:23:40,109 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for b93a997d5d499fc85333e4dccefd8493 in 159ms, sequenceid=113, compaction requested=true 2024-11-07T15:23:40,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:40,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:40,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-07T15:23:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-07T15:23:40,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-07T15:23:40,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 633 msec 2024-11-07T15:23:40,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 645 msec 2024-11-07T15:23:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:40,211 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:40,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:23:40,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/af80c11ec0914e388f9b375c1074dcd8 is 50, key is test_row_0/A:col10/1730993020209/Put/seqid=0 2024-11-07T15:23:40,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742094_1270 (size=14341) 2024-11-07T15:23:40,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993080252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993080252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993080253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993080253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993080259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993080371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993080371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993080372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993080372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993080372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,384 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/7954d864022443c3ae14ead53a57579d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/7954d864022443c3ae14ead53a57579d 2024-11-07T15:23:40,393 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/B of b93a997d5d499fc85333e4dccefd8493 into 7954d864022443c3ae14ead53a57579d(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:40,393 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:40,393 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/B, priority=13, startTime=1730993019917; duration=0sec 2024-11-07T15:23:40,393 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:40,393 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:B 2024-11-07T15:23:40,393 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:23:40,394 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:23:40,394 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/C is initiating minor compaction (all files) 2024-11-07T15:23:40,394 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/C in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:40,394 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/c42044e081d24b098eab7400cd630a38, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/98c2e12a72c04862b6da9451063b5b4a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/5f7f4b19021f4a7ab5f7b4d830a91e1b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/10e329d0b6414e49b7af99af272ff504] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=47.0 K 2024-11-07T15:23:40,395 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting c42044e081d24b098eab7400cd630a38, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1730993016554 2024-11-07T15:23:40,396 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 98c2e12a72c04862b6da9451063b5b4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1730993016710 2024-11-07T15:23:40,397 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f7f4b19021f4a7ab5f7b4d830a91e1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730993017879 2024-11-07T15:23:40,397 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 10e329d0b6414e49b7af99af272ff504, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1730993019050 2024-11-07T15:23:40,406 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/afb4ad3123f3479a806e3088886f0c47 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/afb4ad3123f3479a806e3088886f0c47 2024-11-07T15:23:40,411 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/A of b93a997d5d499fc85333e4dccefd8493 into afb4ad3123f3479a806e3088886f0c47(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:40,411 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:40,411 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/A, priority=13, startTime=1730993019917; duration=0sec 2024-11-07T15:23:40,411 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:40,411 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:A 2024-11-07T15:23:40,412 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#C#compaction#221 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:40,412 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/6ecbe067912344c4bc197a36e91c6763 is 50, key is test_row_0/C:col10/1730993019055/Put/seqid=0 2024-11-07T15:23:40,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742095_1271 (size=12241) 2024-11-07T15:23:40,430 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/6ecbe067912344c4bc197a36e91c6763 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/6ecbe067912344c4bc197a36e91c6763 2024-11-07T15:23:40,439 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/C of b93a997d5d499fc85333e4dccefd8493 into 6ecbe067912344c4bc197a36e91c6763(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:40,439 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:40,439 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/C, priority=12, startTime=1730993019917; duration=0sec 2024-11-07T15:23:40,439 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:40,439 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:C 2024-11-07T15:23:40,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993080581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993080581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993080581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993080581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993080584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T15:23:40,611 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:40,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-07T15:23:40,613 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T15:23:40,614 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:40,614 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:40,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/af80c11ec0914e388f9b375c1074dcd8 2024-11-07T15:23:40,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/9d24b296821344b5b5ee61c799938911 is 50, key is test_row_0/B:col10/1730993020209/Put/seqid=0 2024-11-07T15:23:40,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742096_1272 (size=12001) 2024-11-07T15:23:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T15:23:40,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T15:23:40,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:40,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:40,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:40,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:40,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:40,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:40,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993080891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993080891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993080891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993080891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:40,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993080891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:40,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:40,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:40,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T15:23:40,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:40,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:40,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:40,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:40,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T15:23:41,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/9d24b296821344b5b5ee61c799938911 2024-11-07T15:23:41,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/92ce31d7ebcb4205a6a77cfce54bad3f is 50, key is test_row_0/C:col10/1730993020209/Put/seqid=0 2024-11-07T15:23:41,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742097_1273 (size=12001) 2024-11-07T15:23:41,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T15:23:41,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:41,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:41,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:41,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:41,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:41,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T15:23:41,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:41,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:41,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:41,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:41,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:41,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:41,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T15:23:41,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T15:23:41,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:41,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:41,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:41,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:41,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:41,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993081400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:41,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993081400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:41,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:41,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993081401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:41,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:41,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:41,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993081411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993081411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:41,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:41,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:41,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:41,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:41,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:41,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:41,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/92ce31d7ebcb4205a6a77cfce54bad3f 2024-11-07T15:23:41,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/af80c11ec0914e388f9b375c1074dcd8 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/af80c11ec0914e388f9b375c1074dcd8 2024-11-07T15:23:41,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/af80c11ec0914e388f9b375c1074dcd8, entries=200, sequenceid=124, filesize=14.0 K 2024-11-07T15:23:41,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/9d24b296821344b5b5ee61c799938911 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/9d24b296821344b5b5ee61c799938911 2024-11-07T15:23:41,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/9d24b296821344b5b5ee61c799938911, entries=150, sequenceid=124, filesize=11.7 K 2024-11-07T15:23:41,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/92ce31d7ebcb4205a6a77cfce54bad3f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/92ce31d7ebcb4205a6a77cfce54bad3f 2024-11-07T15:23:41,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/92ce31d7ebcb4205a6a77cfce54bad3f, entries=150, sequenceid=124, filesize=11.7 K 2024-11-07T15:23:41,483 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-07T15:23:41,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for b93a997d5d499fc85333e4dccefd8493 in 1273ms, sequenceid=124, compaction requested=true 2024-11-07T15:23:41,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:41,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:41,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:41,484 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:41,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:41,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:41,484 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:41,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:41,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:41,485 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:41,485 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:41,485 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/B is initiating minor compaction (all files) 2024-11-07T15:23:41,485 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/A is initiating minor compaction (all files) 2024-11-07T15:23:41,485 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/B in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:41,485 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/A in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:41,485 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/7954d864022443c3ae14ead53a57579d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/16263a8c967d449fbacb68d195621124, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/9d24b296821344b5b5ee61c799938911] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=35.4 K 2024-11-07T15:23:41,485 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/afb4ad3123f3479a806e3088886f0c47, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/5dd6bb70a080496886f382fff38ed63b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/af80c11ec0914e388f9b375c1074dcd8] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=37.6 K 2024-11-07T15:23:41,486 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting afb4ad3123f3479a806e3088886f0c47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730993017879 2024-11-07T15:23:41,486 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 7954d864022443c3ae14ead53a57579d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730993017879 2024-11-07T15:23:41,487 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dd6bb70a080496886f382fff38ed63b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1730993019050 2024-11-07T15:23:41,487 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 16263a8c967d449fbacb68d195621124, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1730993019050 2024-11-07T15:23:41,487 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting af80c11ec0914e388f9b375c1074dcd8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1730993020205 2024-11-07T15:23:41,487 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d24b296821344b5b5ee61c799938911, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1730993020205 2024-11-07T15:23:41,498 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#B#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:41,499 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/30cc4f766dee4aa496cb56a320e9529c is 50, key is test_row_0/B:col10/1730993020209/Put/seqid=0 2024-11-07T15:23:41,507 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:41,508 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/99d228a2303c4b6d8cd22d428916f62d is 50, key is test_row_0/A:col10/1730993020209/Put/seqid=0 2024-11-07T15:23:41,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742098_1274 (size=12309) 2024-11-07T15:23:41,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742099_1275 (size=12309) 2024-11-07T15:23:41,528 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/99d228a2303c4b6d8cd22d428916f62d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/99d228a2303c4b6d8cd22d428916f62d 2024-11-07T15:23:41,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T15:23:41,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:41,535 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T15:23:41,535 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/A of b93a997d5d499fc85333e4dccefd8493 into 99d228a2303c4b6d8cd22d428916f62d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:41,535 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:41,536 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/A, priority=13, startTime=1730993021484; duration=0sec 2024-11-07T15:23:41,536 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:41,537 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:A 2024-11-07T15:23:41,537 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-07T15:23:41,538 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-07T15:23:41,539 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-07T15:23:41,539 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. because compaction request was cancelled 2024-11-07T15:23:41,539 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:C 2024-11-07T15:23:41,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/c543cc96486e48b38711e2e1794a19d5 is 50, key is test_row_0/A:col10/1730993020251/Put/seqid=0 2024-11-07T15:23:41,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742100_1276 (size=12151) 2024-11-07T15:23:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T15:23:41,930 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/30cc4f766dee4aa496cb56a320e9529c as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/30cc4f766dee4aa496cb56a320e9529c 2024-11-07T15:23:41,948 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/B of b93a997d5d499fc85333e4dccefd8493 into 30cc4f766dee4aa496cb56a320e9529c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:41,948 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:41,949 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/B, priority=13, startTime=1730993021484; duration=0sec 2024-11-07T15:23:41,949 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:41,949 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:B 2024-11-07T15:23:41,953 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/c543cc96486e48b38711e2e1794a19d5 2024-11-07T15:23:41,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/860053da02cc40d48b98cff1b3884f02 is 50, key is test_row_0/B:col10/1730993020251/Put/seqid=0 2024-11-07T15:23:41,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742101_1277 (size=12151) 2024-11-07T15:23:42,001 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/860053da02cc40d48b98cff1b3884f02 2024-11-07T15:23:42,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/012686647fd242cda005e8bb44b1667f is 50, key is test_row_0/C:col10/1730993020251/Put/seqid=0 2024-11-07T15:23:42,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742102_1278 (size=12151) 2024-11-07T15:23:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:42,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:42,423 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/012686647fd242cda005e8bb44b1667f 2024-11-07T15:23:42,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/c543cc96486e48b38711e2e1794a19d5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/c543cc96486e48b38711e2e1794a19d5 2024-11-07T15:23:42,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,434 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/c543cc96486e48b38711e2e1794a19d5, entries=150, sequenceid=153, filesize=11.9 K 2024-11-07T15:23:42,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993082428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993082427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993082429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993082430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993082430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/860053da02cc40d48b98cff1b3884f02 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/860053da02cc40d48b98cff1b3884f02 2024-11-07T15:23:42,442 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/860053da02cc40d48b98cff1b3884f02, entries=150, sequenceid=153, filesize=11.9 K 2024-11-07T15:23:42,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/012686647fd242cda005e8bb44b1667f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/012686647fd242cda005e8bb44b1667f 2024-11-07T15:23:42,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,447 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/012686647fd242cda005e8bb44b1667f, entries=150, sequenceid=153, filesize=11.9 K 2024-11-07T15:23:42,447 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for b93a997d5d499fc85333e4dccefd8493 in 912ms, sequenceid=153, compaction requested=true 2024-11-07T15:23:42,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:42,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:42,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-07T15:23:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-07T15:23:42,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-07T15:23:42,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8360 sec 2024-11-07T15:23:42,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.8420 sec 2024-11-07T15:23:42,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:42,543 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:42,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T15:23:42,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/ce6302b1978447e3bb77bd16c6c260f4 is 50, key is test_row_0/A:col10/1730993022543/Put/seqid=0 2024-11-07T15:23:42,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742103_1279 (size=16931) 2024-11-07T15:23:42,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993082569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993082573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993082574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993082574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993082575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993082681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993082681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993082681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993082691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993082691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T15:23:42,761 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:42,761 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-07T15:23:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T15:23:42,763 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:42,763 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:42,763 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T15:23:42,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993082891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993082891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993082891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:42,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993082900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993082900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:42,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T15:23:42,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:42,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:42,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:42,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:42,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:42,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/ce6302b1978447e3bb77bd16c6c260f4 2024-11-07T15:23:42,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/5e68910146ee444d885f102cf5871e75 is 50, key is test_row_0/B:col10/1730993022543/Put/seqid=0 2024-11-07T15:23:42,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742104_1280 (size=12151) 2024-11-07T15:23:43,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T15:23:43,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:43,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:43,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:43,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:43,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T15:23:43,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993083201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:43,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:43,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993083201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993083201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:43,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993083210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:43,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993083210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:43,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:43,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T15:23:43,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:43,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:43,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:43,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:43,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:43,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/5e68910146ee444d885f102cf5871e75 2024-11-07T15:23:43,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/d5ca8dbea7324ac188d00c0b8b62d540 is 50, key is test_row_0/C:col10/1730993022543/Put/seqid=0 2024-11-07T15:23:43,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T15:23:43,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:43,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:43,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:43,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:43,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:43,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742105_1281 (size=12151) 2024-11-07T15:23:43,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/d5ca8dbea7324ac188d00c0b8b62d540 2024-11-07T15:23:43,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/ce6302b1978447e3bb77bd16c6c260f4 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/ce6302b1978447e3bb77bd16c6c260f4 2024-11-07T15:23:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T15:23:43,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/ce6302b1978447e3bb77bd16c6c260f4, entries=250, sequenceid=168, filesize=16.5 K 2024-11-07T15:23:43,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/5e68910146ee444d885f102cf5871e75 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/5e68910146ee444d885f102cf5871e75 2024-11-07T15:23:43,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/5e68910146ee444d885f102cf5871e75, entries=150, sequenceid=168, filesize=11.9 K 2024-11-07T15:23:43,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/d5ca8dbea7324ac188d00c0b8b62d540 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/d5ca8dbea7324ac188d00c0b8b62d540 2024-11-07T15:23:43,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/d5ca8dbea7324ac188d00c0b8b62d540, entries=150, sequenceid=168, filesize=11.9 K 2024-11-07T15:23:43,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for b93a997d5d499fc85333e4dccefd8493 in 858ms, sequenceid=168, compaction requested=true 2024-11-07T15:23:43,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:43,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:43,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:43,401 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:43,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:43,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:43,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:43,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T15:23:43,401 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:23:43,402 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41391 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:43,402 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/A is initiating minor compaction (all files) 2024-11-07T15:23:43,402 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/A in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:43,402 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/99d228a2303c4b6d8cd22d428916f62d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/c543cc96486e48b38711e2e1794a19d5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/ce6302b1978447e3bb77bd16c6c260f4] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=40.4 K 2024-11-07T15:23:43,403 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99d228a2303c4b6d8cd22d428916f62d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1730993020205 2024-11-07T15:23:43,403 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting c543cc96486e48b38711e2e1794a19d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1730993020250 2024-11-07T15:23:43,404 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:23:43,404 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/C is initiating minor compaction (all files) 2024-11-07T15:23:43,404 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce6302b1978447e3bb77bd16c6c260f4, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730993022425 2024-11-07T15:23:43,404 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/C in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:43,404 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/6ecbe067912344c4bc197a36e91c6763, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/92ce31d7ebcb4205a6a77cfce54bad3f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/012686647fd242cda005e8bb44b1667f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/d5ca8dbea7324ac188d00c0b8b62d540] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=47.4 K 2024-11-07T15:23:43,405 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ecbe067912344c4bc197a36e91c6763, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1730993019050 2024-11-07T15:23:43,405 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 92ce31d7ebcb4205a6a77cfce54bad3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1730993020205 2024-11-07T15:23:43,405 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 012686647fd242cda005e8bb44b1667f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1730993020250 2024-11-07T15:23:43,405 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting d5ca8dbea7324ac188d00c0b8b62d540, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730993022428 2024-11-07T15:23:43,413 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#A#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:43,414 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/ec8527c2a0454d31a1dc61006e88194b is 50, key is test_row_0/A:col10/1730993022543/Put/seqid=0 2024-11-07T15:23:43,435 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#C#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:43,436 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/b2b856d1e4744a78bba999e6cb748892 is 50, key is test_row_0/C:col10/1730993022543/Put/seqid=0 2024-11-07T15:23:43,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742107_1283 (size=12527) 2024-11-07T15:23:43,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742106_1282 (size=12561) 2024-11-07T15:23:43,454 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/ec8527c2a0454d31a1dc61006e88194b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/ec8527c2a0454d31a1dc61006e88194b 2024-11-07T15:23:43,459 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/A of b93a997d5d499fc85333e4dccefd8493 into ec8527c2a0454d31a1dc61006e88194b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:43,459 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:43,459 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/A, priority=13, startTime=1730993023401; duration=0sec 2024-11-07T15:23:43,459 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:43,459 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:A 2024-11-07T15:23:43,459 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:43,461 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:43,461 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/B is initiating minor compaction (all files) 2024-11-07T15:23:43,461 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/B in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:43,461 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/30cc4f766dee4aa496cb56a320e9529c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/860053da02cc40d48b98cff1b3884f02, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/5e68910146ee444d885f102cf5871e75] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=35.8 K 2024-11-07T15:23:43,462 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30cc4f766dee4aa496cb56a320e9529c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1730993020205 2024-11-07T15:23:43,477 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 860053da02cc40d48b98cff1b3884f02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1730993020250 2024-11-07T15:23:43,477 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e68910146ee444d885f102cf5871e75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730993022428 2024-11-07T15:23:43,487 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#B#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:43,487 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/6c0b02b5188c49f48ebb7c91c0f2dd87 is 50, key is test_row_0/B:col10/1730993022543/Put/seqid=0 2024-11-07T15:23:43,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742108_1284 (size=12561) 2024-11-07T15:23:43,533 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/6c0b02b5188c49f48ebb7c91c0f2dd87 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/6c0b02b5188c49f48ebb7c91c0f2dd87 2024-11-07T15:23:43,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T15:23:43,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:43,534 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T15:23:43,539 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/B of b93a997d5d499fc85333e4dccefd8493 into 6c0b02b5188c49f48ebb7c91c0f2dd87(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:43,539 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:43,539 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/B, priority=13, startTime=1730993023401; duration=0sec 2024-11-07T15:23:43,539 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:43,539 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:B 2024-11-07T15:23:43,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/48c00bf7acc54a79ae86eed7aa812252 is 50, key is test_row_0/A:col10/1730993022574/Put/seqid=0 2024-11-07T15:23:43,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742109_1285 (size=12151) 2024-11-07T15:23:43,594 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/48c00bf7acc54a79ae86eed7aa812252 2024-11-07T15:23:43,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/82351bb049a34a529ec35d9741d19ebc is 50, key is test_row_0/B:col10/1730993022574/Put/seqid=0 2024-11-07T15:23:43,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742110_1286 (size=12151) 2024-11-07T15:23:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:43,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:43,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:43,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993083726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993083726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:43,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993083727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:43,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993083730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:43,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993083730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:43,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993083841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:43,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993083841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:43,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993083841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993083841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993083841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:43,848 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/b2b856d1e4744a78bba999e6cb748892 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/b2b856d1e4744a78bba999e6cb748892 2024-11-07T15:23:43,853 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/C of b93a997d5d499fc85333e4dccefd8493 into b2b856d1e4744a78bba999e6cb748892(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:43,853 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:43,853 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/C, priority=12, startTime=1730993023401; duration=0sec 2024-11-07T15:23:43,853 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:43,853 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:C 2024-11-07T15:23:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T15:23:44,011 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/82351bb049a34a529ec35d9741d19ebc 2024-11-07T15:23:44,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/6ee4f3dc62cc414bb192aef81053cdad is 50, key is test_row_0/C:col10/1730993022574/Put/seqid=0 2024-11-07T15:23:44,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742111_1287 (size=12151) 2024-11-07T15:23:44,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:44,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993084051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993084050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:44,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:44,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993084051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993084051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:44,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993084051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993084361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993084361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993084361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993084361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:44,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993084361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:44,433 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/6ee4f3dc62cc414bb192aef81053cdad 2024-11-07T15:23:44,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/48c00bf7acc54a79ae86eed7aa812252 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/48c00bf7acc54a79ae86eed7aa812252 2024-11-07T15:23:44,441 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/48c00bf7acc54a79ae86eed7aa812252, entries=150, sequenceid=192, filesize=11.9 K 2024-11-07T15:23:44,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/82351bb049a34a529ec35d9741d19ebc as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/82351bb049a34a529ec35d9741d19ebc 2024-11-07T15:23:44,447 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/82351bb049a34a529ec35d9741d19ebc, entries=150, sequenceid=192, filesize=11.9 K 2024-11-07T15:23:44,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/6ee4f3dc62cc414bb192aef81053cdad as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/6ee4f3dc62cc414bb192aef81053cdad 2024-11-07T15:23:44,452 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/6ee4f3dc62cc414bb192aef81053cdad, entries=150, sequenceid=192, filesize=11.9 K 2024-11-07T15:23:44,453 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for b93a997d5d499fc85333e4dccefd8493 in 919ms, sequenceid=192, compaction requested=false 2024-11-07T15:23:44,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:44,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:44,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-07T15:23:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-07T15:23:44,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-07T15:23:44,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6910 sec 2024-11-07T15:23:44,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.6960 sec 2024-11-07T15:23:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:44,872 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:44,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T15:23:44,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/346b3be6483845d1bf654ad7d6ab86c3 is 50, key is test_row_0/A:col10/1730993023726/Put/seqid=0 2024-11-07T15:23:44,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742112_1288 (size=12151) 2024-11-07T15:23:44,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:44,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993084897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:44,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:44,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993084900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993084901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:44,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:44,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993084901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:44,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:44,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993084902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:44,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T15:23:44,911 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:44,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:44,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-07T15:23:44,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T15:23:44,913 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:44,914 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:44,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:45,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:45,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993085010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:45,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:45,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993085010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:45,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:45,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993085011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993085011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993085011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:45,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T15:23:45,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T15:23:45,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:45,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:45,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:45,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:45,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:45,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:45,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T15:23:45,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:45,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:45,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:45,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:45,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:45,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:45,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993085221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993085221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:45,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:45,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993085221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993085221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:45,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993085222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T15:23:45,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/346b3be6483845d1bf654ad7d6ab86c3 2024-11-07T15:23:45,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/209dc81ec1a84562b4fc5a378aa018b6 is 50, key is test_row_0/B:col10/1730993023726/Put/seqid=0 2024-11-07T15:23:45,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742113_1289 (size=12151) 2024-11-07T15:23:45,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/209dc81ec1a84562b4fc5a378aa018b6 2024-11-07T15:23:45,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/1279435b4d4e40b79fa3ff6acbb844ef is 50, key is test_row_0/C:col10/1730993023726/Put/seqid=0 2024-11-07T15:23:45,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742114_1290 (size=12151) 2024-11-07T15:23:45,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T15:23:45,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:45,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:45,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:45,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:45,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:45,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:45,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T15:23:45,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:45,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:45,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:45,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:45,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:45,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993085531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993085531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:45,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993085531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993085531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:45,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993085531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T15:23:45,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T15:23:45,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:45,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:45,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:45,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:45,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:45,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/1279435b4d4e40b79fa3ff6acbb844ef 2024-11-07T15:23:45,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/346b3be6483845d1bf654ad7d6ab86c3 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/346b3be6483845d1bf654ad7d6ab86c3 2024-11-07T15:23:45,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/346b3be6483845d1bf654ad7d6ab86c3, entries=150, sequenceid=210, filesize=11.9 K 2024-11-07T15:23:45,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/209dc81ec1a84562b4fc5a378aa018b6 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/209dc81ec1a84562b4fc5a378aa018b6 2024-11-07T15:23:45,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/209dc81ec1a84562b4fc5a378aa018b6, entries=150, sequenceid=210, filesize=11.9 K 2024-11-07T15:23:45,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/1279435b4d4e40b79fa3ff6acbb844ef as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/1279435b4d4e40b79fa3ff6acbb844ef 2024-11-07T15:23:45,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/1279435b4d4e40b79fa3ff6acbb844ef, entries=150, sequenceid=210, filesize=11.9 K 2024-11-07T15:23:45,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for b93a997d5d499fc85333e4dccefd8493 in 874ms, sequenceid=210, compaction requested=true 2024-11-07T15:23:45,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:45,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:45,746 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:45,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:45,746 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:45,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:45,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:45,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:45,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:45,747 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:45,747 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/A is initiating minor compaction (all files) 2024-11-07T15:23:45,748 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/A in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:45,748 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/ec8527c2a0454d31a1dc61006e88194b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/48c00bf7acc54a79ae86eed7aa812252, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/346b3be6483845d1bf654ad7d6ab86c3] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=36.0 K 2024-11-07T15:23:45,748 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:45,748 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/B is initiating minor compaction (all files) 2024-11-07T15:23:45,748 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/B in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:45,748 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/6c0b02b5188c49f48ebb7c91c0f2dd87, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/82351bb049a34a529ec35d9741d19ebc, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/209dc81ec1a84562b4fc5a378aa018b6] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=36.0 K 2024-11-07T15:23:45,749 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c0b02b5188c49f48ebb7c91c0f2dd87, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730993022428 2024-11-07T15:23:45,749 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec8527c2a0454d31a1dc61006e88194b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730993022428 2024-11-07T15:23:45,749 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 82351bb049a34a529ec35d9741d19ebc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1730993022558 2024-11-07T15:23:45,749 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48c00bf7acc54a79ae86eed7aa812252, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1730993022558 2024-11-07T15:23:45,750 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 209dc81ec1a84562b4fc5a378aa018b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730993023726 2024-11-07T15:23:45,750 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 346b3be6483845d1bf654ad7d6ab86c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730993023726 2024-11-07T15:23:45,771 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#B#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:45,771 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#A#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:45,771 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/5c0c7f7a6f254338a8e872af8f637dde is 50, key is test_row_0/B:col10/1730993023726/Put/seqid=0 2024-11-07T15:23:45,771 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/65bd3751cfef4d69a8113441557a8f82 is 50, key is test_row_0/A:col10/1730993023726/Put/seqid=0 2024-11-07T15:23:45,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742115_1291 (size=12663) 2024-11-07T15:23:45,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742116_1292 (size=12663) 2024-11-07T15:23:45,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T15:23:45,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:45,831 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T15:23:45,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/06049217041643fab8fc1cb259f6dd37 is 50, key is test_row_0/A:col10/1730993024886/Put/seqid=0 2024-11-07T15:23:45,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742117_1293 (size=12151) 2024-11-07T15:23:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:46,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T15:23:46,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993086055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993086054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993086059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993086060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993086060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993086171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993086171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993086171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993086171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993086171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,188 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/5c0c7f7a6f254338a8e872af8f637dde as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/5c0c7f7a6f254338a8e872af8f637dde 2024-11-07T15:23:46,196 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/B of b93a997d5d499fc85333e4dccefd8493 into 5c0c7f7a6f254338a8e872af8f637dde(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:46,196 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:46,196 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/B, priority=13, startTime=1730993025746; duration=0sec 2024-11-07T15:23:46,196 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:46,196 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:B 2024-11-07T15:23:46,196 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:46,200 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:46,200 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/C is initiating minor compaction (all files) 2024-11-07T15:23:46,200 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/C in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:46,200 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/b2b856d1e4744a78bba999e6cb748892, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/6ee4f3dc62cc414bb192aef81053cdad, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/1279435b4d4e40b79fa3ff6acbb844ef] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=36.0 K 2024-11-07T15:23:46,201 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting b2b856d1e4744a78bba999e6cb748892, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730993022428 2024-11-07T15:23:46,201 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ee4f3dc62cc414bb192aef81053cdad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1730993022558 2024-11-07T15:23:46,201 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 1279435b4d4e40b79fa3ff6acbb844ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730993023726 2024-11-07T15:23:46,201 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/65bd3751cfef4d69a8113441557a8f82 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/65bd3751cfef4d69a8113441557a8f82 2024-11-07T15:23:46,207 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/A of b93a997d5d499fc85333e4dccefd8493 into 65bd3751cfef4d69a8113441557a8f82(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:46,207 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:46,207 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/A, priority=13, startTime=1730993025746; duration=0sec 2024-11-07T15:23:46,207 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:46,207 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:A 2024-11-07T15:23:46,209 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#C#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:46,209 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/b104ed28eb45484c92e521a14cef0ef0 is 50, key is test_row_0/C:col10/1730993023726/Put/seqid=0 2024-11-07T15:23:46,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742118_1294 (size=12629) 2024-11-07T15:23:46,262 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/06049217041643fab8fc1cb259f6dd37 2024-11-07T15:23:46,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/076c0a8686b64c359093737c275aa975 is 50, key is test_row_0/B:col10/1730993024886/Put/seqid=0 2024-11-07T15:23:46,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742119_1295 (size=12151) 2024-11-07T15:23:46,280 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/076c0a8686b64c359093737c275aa975 2024-11-07T15:23:46,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/f58cc622cc0e44509c2ce1ebe7ccef1b is 50, key is test_row_0/C:col10/1730993024886/Put/seqid=0 2024-11-07T15:23:46,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742120_1296 (size=12151) 2024-11-07T15:23:46,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993086381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993086381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993086381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993086381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993086382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,625 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/b104ed28eb45484c92e521a14cef0ef0 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/b104ed28eb45484c92e521a14cef0ef0 2024-11-07T15:23:46,629 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/C of b93a997d5d499fc85333e4dccefd8493 into b104ed28eb45484c92e521a14cef0ef0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:46,629 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:46,629 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/C, priority=13, startTime=1730993025746; duration=0sec 2024-11-07T15:23:46,629 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:46,629 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:C 2024-11-07T15:23:46,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993086691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993086691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993086691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993086692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993086697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:46,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:46,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:46,709 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/f58cc622cc0e44509c2ce1ebe7ccef1b 2024-11-07T15:23:46,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/06049217041643fab8fc1cb259f6dd37 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/06049217041643fab8fc1cb259f6dd37 2024-11-07T15:23:46,717 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/06049217041643fab8fc1cb259f6dd37, entries=150, sequenceid=230, filesize=11.9 K 2024-11-07T15:23:46,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/076c0a8686b64c359093737c275aa975 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/076c0a8686b64c359093737c275aa975 2024-11-07T15:23:46,721 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/076c0a8686b64c359093737c275aa975, entries=150, sequenceid=230, filesize=11.9 K 2024-11-07T15:23:46,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/f58cc622cc0e44509c2ce1ebe7ccef1b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/f58cc622cc0e44509c2ce1ebe7ccef1b 2024-11-07T15:23:46,725 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/f58cc622cc0e44509c2ce1ebe7ccef1b, entries=150, sequenceid=230, filesize=11.9 K 2024-11-07T15:23:46,726 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for b93a997d5d499fc85333e4dccefd8493 in 895ms, sequenceid=230, compaction requested=false 2024-11-07T15:23:46,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:46,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:46,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-07T15:23:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-07T15:23:46,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-07T15:23:46,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8130 sec 2024-11-07T15:23:46,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.8170 sec 2024-11-07T15:23:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T15:23:47,061 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:47,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:47,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-07T15:23:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T15:23:47,064 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:47,065 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:47,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T15:23:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:47,202 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:47,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-07T15:23:47,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/7a611ac2e8374795af3795290659fe47 is 50, key is test_row_0/A:col10/1730993026059/Put/seqid=0 2024-11-07T15:23:47,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T15:23:47,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:47,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:47,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:47,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:47,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:47,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993087225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993087233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993087233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993087233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993087234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742121_1297 (size=12151) 2024-11-07T15:23:47,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/7a611ac2e8374795af3795290659fe47 2024-11-07T15:23:47,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/93334fe4bcef4e908dd9643ff95c4c1c is 50, key is test_row_0/B:col10/1730993026059/Put/seqid=0 2024-11-07T15:23:47,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742122_1298 (size=12151) 2024-11-07T15:23:47,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/93334fe4bcef4e908dd9643ff95c4c1c 2024-11-07T15:23:47,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/4a0a25bb3be44b17866daaee87c7b8b2 is 50, key is test_row_0/C:col10/1730993026059/Put/seqid=0 2024-11-07T15:23:47,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742123_1299 (size=12151) 2024-11-07T15:23:47,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993087340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993087341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993087341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993087341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993087341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T15:23:47,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:47,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:47,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:47,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:47,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T15:23:47,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T15:23:47,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:47,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:47,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:47,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:47,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:47,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993087551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993087551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993087551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993087551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993087551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/4a0a25bb3be44b17866daaee87c7b8b2 2024-11-07T15:23:47,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/7a611ac2e8374795af3795290659fe47 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/7a611ac2e8374795af3795290659fe47 2024-11-07T15:23:47,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/7a611ac2e8374795af3795290659fe47, entries=150, sequenceid=250, filesize=11.9 K 2024-11-07T15:23:47,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/93334fe4bcef4e908dd9643ff95c4c1c as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/93334fe4bcef4e908dd9643ff95c4c1c 2024-11-07T15:23:47,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/93334fe4bcef4e908dd9643ff95c4c1c, entries=150, sequenceid=250, filesize=11.9 K 2024-11-07T15:23:47,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/4a0a25bb3be44b17866daaee87c7b8b2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/4a0a25bb3be44b17866daaee87c7b8b2 2024-11-07T15:23:47,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T15:23:47,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:47,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:47,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:47,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:47,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T15:23:47,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/4a0a25bb3be44b17866daaee87c7b8b2, entries=150, sequenceid=250, filesize=11.9 K 2024-11-07T15:23:47,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for b93a997d5d499fc85333e4dccefd8493 in 492ms, sequenceid=250, compaction requested=true 2024-11-07T15:23:47,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:47,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:47,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:47,694 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:47,694 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:47,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:47,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:47,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:47,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:47,695 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:47,695 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:47,695 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/A is initiating minor compaction (all files) 2024-11-07T15:23:47,695 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/B is initiating minor compaction (all files) 2024-11-07T15:23:47,695 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/A in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:47,695 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/B in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:47,695 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/5c0c7f7a6f254338a8e872af8f637dde, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/076c0a8686b64c359093737c275aa975, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/93334fe4bcef4e908dd9643ff95c4c1c] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=36.1 K 2024-11-07T15:23:47,695 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/65bd3751cfef4d69a8113441557a8f82, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/06049217041643fab8fc1cb259f6dd37, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/7a611ac2e8374795af3795290659fe47] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=36.1 K 2024-11-07T15:23:47,696 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65bd3751cfef4d69a8113441557a8f82, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730993023726 2024-11-07T15:23:47,696 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c0c7f7a6f254338a8e872af8f637dde, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730993023726 2024-11-07T15:23:47,696 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06049217041643fab8fc1cb259f6dd37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1730993024886 2024-11-07T15:23:47,696 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 076c0a8686b64c359093737c275aa975, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1730993024886 2024-11-07T15:23:47,696 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a611ac2e8374795af3795290659fe47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1730993026053 2024-11-07T15:23:47,696 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 93334fe4bcef4e908dd9643ff95c4c1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1730993026053 2024-11-07T15:23:47,706 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:47,706 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/57e8f80db5b040489af5a0b24b3796d1 is 50, key is test_row_0/B:col10/1730993026059/Put/seqid=0 2024-11-07T15:23:47,710 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#A#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:47,710 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/df974b9a66e04c40a1496784c933bfc5 is 50, key is test_row_0/A:col10/1730993026059/Put/seqid=0 2024-11-07T15:23:47,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742124_1300 (size=12765) 2024-11-07T15:23:47,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742125_1301 (size=12765) 2024-11-07T15:23:47,742 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/df974b9a66e04c40a1496784c933bfc5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/df974b9a66e04c40a1496784c933bfc5 2024-11-07T15:23:47,747 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/A of b93a997d5d499fc85333e4dccefd8493 into df974b9a66e04c40a1496784c933bfc5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:47,747 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:47,747 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/A, priority=13, startTime=1730993027694; duration=0sec 2024-11-07T15:23:47,747 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:47,747 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:A 2024-11-07T15:23:47,747 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:47,749 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:47,749 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/C is initiating minor compaction (all files) 2024-11-07T15:23:47,749 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/C in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:47,749 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/b104ed28eb45484c92e521a14cef0ef0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/f58cc622cc0e44509c2ce1ebe7ccef1b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/4a0a25bb3be44b17866daaee87c7b8b2] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=36.1 K 2024-11-07T15:23:47,750 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting b104ed28eb45484c92e521a14cef0ef0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1730993023726 2024-11-07T15:23:47,750 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting f58cc622cc0e44509c2ce1ebe7ccef1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1730993024886 2024-11-07T15:23:47,750 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a0a25bb3be44b17866daaee87c7b8b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1730993026053 2024-11-07T15:23:47,778 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#C#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:47,779 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/bf71b7ce593142c5ab64a83effb9e788 is 50, key is test_row_0/C:col10/1730993026059/Put/seqid=0 2024-11-07T15:23:47,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742126_1302 (size=12731) 2024-11-07T15:23:47,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T15:23:47,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:47,842 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T15:23:47,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/29510235f92f4cb8a0578384e2905309 is 50, key is test_row_0/A:col10/1730993027225/Put/seqid=0 2024-11-07T15:23:47,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742127_1303 (size=12301) 2024-11-07T15:23:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:47,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:47,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993087874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993087878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993087878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993087879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993087880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993087991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993087991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993087991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993087991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:47,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993087992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:48,137 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/57e8f80db5b040489af5a0b24b3796d1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/57e8f80db5b040489af5a0b24b3796d1 2024-11-07T15:23:48,141 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/B of b93a997d5d499fc85333e4dccefd8493 into 57e8f80db5b040489af5a0b24b3796d1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:48,141 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:48,142 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/B, priority=13, startTime=1730993027694; duration=0sec 2024-11-07T15:23:48,142 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:48,142 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:B 2024-11-07T15:23:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T15:23:48,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993088201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:48,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993088202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993088202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993088202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993088202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:48,205 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/bf71b7ce593142c5ab64a83effb9e788 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/bf71b7ce593142c5ab64a83effb9e788 2024-11-07T15:23:48,209 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/C of b93a997d5d499fc85333e4dccefd8493 into bf71b7ce593142c5ab64a83effb9e788(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:48,209 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:48,209 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/C, priority=13, startTime=1730993027694; duration=0sec 2024-11-07T15:23:48,209 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:48,209 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:C 2024-11-07T15:23:48,256 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/29510235f92f4cb8a0578384e2905309 2024-11-07T15:23:48,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/367fd6b2859247a9b6e16e5911768ac4 is 50, key is test_row_0/B:col10/1730993027225/Put/seqid=0 2024-11-07T15:23:48,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742128_1304 (size=12301) 2024-11-07T15:23:48,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993088510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:48,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993088511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:48,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993088511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:48,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:48,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993088511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:48,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:48,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:48,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993088511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:48,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:48,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:48,667 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/367fd6b2859247a9b6e16e5911768ac4 2024-11-07T15:23:48,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/0db2dc769be54e2c9b5a13224dfef4a4 is 50, key is test_row_0/C:col10/1730993027225/Put/seqid=0 2024-11-07T15:23:48,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742129_1305 (size=12301) 2024-11-07T15:23:49,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993089020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:49,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:49,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:49,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993089021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:49,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:49,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993089021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:49,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:49,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:49,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:49,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993089021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:49,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:49,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:49,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:49,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993089021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:49,082 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/0db2dc769be54e2c9b5a13224dfef4a4 2024-11-07T15:23:49,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/29510235f92f4cb8a0578384e2905309 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/29510235f92f4cb8a0578384e2905309 2024-11-07T15:23:49,094 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/29510235f92f4cb8a0578384e2905309, entries=150, sequenceid=270, filesize=12.0 K 2024-11-07T15:23:49,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/367fd6b2859247a9b6e16e5911768ac4 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/367fd6b2859247a9b6e16e5911768ac4 2024-11-07T15:23:49,098 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/367fd6b2859247a9b6e16e5911768ac4, entries=150, sequenceid=270, filesize=12.0 K 2024-11-07T15:23:49,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/0db2dc769be54e2c9b5a13224dfef4a4 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/0db2dc769be54e2c9b5a13224dfef4a4 2024-11-07T15:23:49,108 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/0db2dc769be54e2c9b5a13224dfef4a4, entries=150, sequenceid=270, filesize=12.0 K 2024-11-07T15:23:49,109 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for b93a997d5d499fc85333e4dccefd8493 in 1267ms, sequenceid=270, compaction requested=false 2024-11-07T15:23:49,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:49,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:49,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-07T15:23:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-07T15:23:49,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-07T15:23:49,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0460 sec 2024-11-07T15:23:49,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.0520 sec 2024-11-07T15:23:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T15:23:49,220 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:49,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:49,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-07T15:23:49,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T15:23:49,223 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:49,224 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:49,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:49,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T15:23:49,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T15:23:49,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:49,379 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-07T15:23:49,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/d3b8372fd18f4cbf8080ddcb66fb94bd is 50, key is test_row_0/A:col10/1730993027879/Put/seqid=0 2024-11-07T15:23:49,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742130_1306 (size=12301) 2024-11-07T15:23:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T15:23:49,811 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/d3b8372fd18f4cbf8080ddcb66fb94bd 2024-11-07T15:23:49,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/3710a38e222a46c5a41d36fe5795ee70 is 50, key is test_row_0/B:col10/1730993027879/Put/seqid=0 2024-11-07T15:23:49,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742131_1307 (size=12301) 2024-11-07T15:23:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T15:23:49,857 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/3710a38e222a46c5a41d36fe5795ee70 2024-11-07T15:23:49,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/647435839ffb45d09464d011bbe5f61e is 50, key is test_row_0/C:col10/1730993027879/Put/seqid=0 2024-11-07T15:23:49,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742132_1308 (size=12301) 2024-11-07T15:23:49,879 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/647435839ffb45d09464d011bbe5f61e 2024-11-07T15:23:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/d3b8372fd18f4cbf8080ddcb66fb94bd as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/d3b8372fd18f4cbf8080ddcb66fb94bd 2024-11-07T15:23:49,892 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/d3b8372fd18f4cbf8080ddcb66fb94bd, entries=150, sequenceid=289, filesize=12.0 K 2024-11-07T15:23:49,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/3710a38e222a46c5a41d36fe5795ee70 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/3710a38e222a46c5a41d36fe5795ee70 2024-11-07T15:23:49,897 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/3710a38e222a46c5a41d36fe5795ee70, entries=150, sequenceid=289, filesize=12.0 K 2024-11-07T15:23:49,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/647435839ffb45d09464d011bbe5f61e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/647435839ffb45d09464d011bbe5f61e 2024-11-07T15:23:49,902 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/647435839ffb45d09464d011bbe5f61e, entries=150, sequenceid=289, filesize=12.0 K 2024-11-07T15:23:49,903 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for b93a997d5d499fc85333e4dccefd8493 in 524ms, sequenceid=289, compaction requested=true 2024-11-07T15:23:49,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:49,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:49,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-07T15:23:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-07T15:23:49,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-07T15:23:49,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 682 msec 2024-11-07T15:23:49,912 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 689 msec 2024-11-07T15:23:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:50,041 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:50,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T15:23:50,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/631b0b6c8aee416ea22e549f7778f93d is 50, key is test_row_0/A:col10/1730993030038/Put/seqid=0 2024-11-07T15:23:50,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742133_1309 (size=12301) 2024-11-07T15:23:50,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993090067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993090068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993090072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993090072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993090073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993090181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993090181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993090181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993090181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993090182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T15:23:50,361 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:50,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:50,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-07T15:23:50,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T15:23:50,364 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:50,364 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:50,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:50,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993090390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993090390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993090391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993090391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993090391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/631b0b6c8aee416ea22e549f7778f93d 2024-11-07T15:23:50,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/4c48f56db1eb449389af6c377687bcb4 is 50, key is test_row_0/B:col10/1730993030038/Put/seqid=0 2024-11-07T15:23:50,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742134_1310 (size=12301) 2024-11-07T15:23:50,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/4c48f56db1eb449389af6c377687bcb4 2024-11-07T15:23:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T15:23:50,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/8c9ddcb57bbb41b884e2f5989cb56d24 is 50, key is test_row_0/C:col10/1730993030038/Put/seqid=0 2024-11-07T15:23:50,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742135_1311 (size=12301) 2024-11-07T15:23:50,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/8c9ddcb57bbb41b884e2f5989cb56d24 2024-11-07T15:23:50,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/631b0b6c8aee416ea22e549f7778f93d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/631b0b6c8aee416ea22e549f7778f93d 2024-11-07T15:23:50,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/631b0b6c8aee416ea22e549f7778f93d, entries=150, sequenceid=301, filesize=12.0 K 2024-11-07T15:23:50,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/4c48f56db1eb449389af6c377687bcb4 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/4c48f56db1eb449389af6c377687bcb4 2024-11-07T15:23:50,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/4c48f56db1eb449389af6c377687bcb4, entries=150, sequenceid=301, filesize=12.0 K 2024-11-07T15:23:50,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/8c9ddcb57bbb41b884e2f5989cb56d24 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/8c9ddcb57bbb41b884e2f5989cb56d24 2024-11-07T15:23:50,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/8c9ddcb57bbb41b884e2f5989cb56d24, entries=150, sequenceid=301, filesize=12.0 K 2024-11-07T15:23:50,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b93a997d5d499fc85333e4dccefd8493 in 469ms, sequenceid=301, compaction requested=true 2024-11-07T15:23:50,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:50,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:50,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:50,510 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:23:50,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:50,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:50,510 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:23:50,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:50,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:50,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T15:23:50,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:50,519 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T15:23:50,522 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:23:50,522 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:23:50,522 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/A is initiating minor compaction (all files) 2024-11-07T15:23:50,522 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/B is initiating minor compaction (all files) 2024-11-07T15:23:50,522 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/B in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:50,522 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/A in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:50,522 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/df974b9a66e04c40a1496784c933bfc5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/29510235f92f4cb8a0578384e2905309, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/d3b8372fd18f4cbf8080ddcb66fb94bd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/631b0b6c8aee416ea22e549f7778f93d] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=48.5 K 2024-11-07T15:23:50,522 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/57e8f80db5b040489af5a0b24b3796d1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/367fd6b2859247a9b6e16e5911768ac4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/3710a38e222a46c5a41d36fe5795ee70, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/4c48f56db1eb449389af6c377687bcb4] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=48.5 K 2024-11-07T15:23:50,523 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting df974b9a66e04c40a1496784c933bfc5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1730993026053 2024-11-07T15:23:50,523 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e8f80db5b040489af5a0b24b3796d1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1730993026053 2024-11-07T15:23:50,523 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29510235f92f4cb8a0578384e2905309, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1730993027216 2024-11-07T15:23:50,524 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 367fd6b2859247a9b6e16e5911768ac4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1730993027216 2024-11-07T15:23:50,524 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3b8372fd18f4cbf8080ddcb66fb94bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730993027872 2024-11-07T15:23:50,525 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 631b0b6c8aee416ea22e549f7778f93d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1730993030038 2024-11-07T15:23:50,525 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 3710a38e222a46c5a41d36fe5795ee70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730993027872 2024-11-07T15:23:50,526 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c48f56db1eb449389af6c377687bcb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1730993030038 2024-11-07T15:23:50,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/6101f2f1ae72407db882e6929cb716d9 is 50, key is test_row_0/A:col10/1730993030070/Put/seqid=0 2024-11-07T15:23:50,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742136_1312 (size=12301) 2024-11-07T15:23:50,547 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#A#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:50,547 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/c0dc3294e88d42c9a2b1885a21ef4c53 is 50, key is test_row_0/A:col10/1730993030038/Put/seqid=0 2024-11-07T15:23:50,551 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#B#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:50,552 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/a5bd3c23bc69487d95248bc9e769df0e is 50, key is test_row_0/B:col10/1730993030038/Put/seqid=0 2024-11-07T15:23:50,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742137_1313 (size=13051) 2024-11-07T15:23:50,574 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/c0dc3294e88d42c9a2b1885a21ef4c53 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/c0dc3294e88d42c9a2b1885a21ef4c53 2024-11-07T15:23:50,580 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/A of b93a997d5d499fc85333e4dccefd8493 into c0dc3294e88d42c9a2b1885a21ef4c53(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:50,581 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:50,581 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/A, priority=12, startTime=1730993030510; duration=0sec 2024-11-07T15:23:50,581 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:50,581 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:A 2024-11-07T15:23:50,581 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:23:50,583 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:23:50,583 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/C is initiating minor compaction (all files) 2024-11-07T15:23:50,583 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/C in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:50,583 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/bf71b7ce593142c5ab64a83effb9e788, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/0db2dc769be54e2c9b5a13224dfef4a4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/647435839ffb45d09464d011bbe5f61e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/8c9ddcb57bbb41b884e2f5989cb56d24] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=48.5 K 2024-11-07T15:23:50,584 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf71b7ce593142c5ab64a83effb9e788, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1730993026053 2024-11-07T15:23:50,584 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0db2dc769be54e2c9b5a13224dfef4a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1730993027216 2024-11-07T15:23:50,584 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 647435839ffb45d09464d011bbe5f61e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730993027872 2024-11-07T15:23:50,585 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c9ddcb57bbb41b884e2f5989cb56d24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1730993030038 2024-11-07T15:23:50,596 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#C#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:50,596 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/21f08385f85643ad8aacc373f0aded34 is 50, key is test_row_0/C:col10/1730993030038/Put/seqid=0 2024-11-07T15:23:50,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742138_1314 (size=13051) 2024-11-07T15:23:50,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742139_1315 (size=13017) 2024-11-07T15:23:50,615 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/21f08385f85643ad8aacc373f0aded34 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/21f08385f85643ad8aacc373f0aded34 2024-11-07T15:23:50,620 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/C of b93a997d5d499fc85333e4dccefd8493 into 21f08385f85643ad8aacc373f0aded34(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:50,620 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:50,620 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/C, priority=12, startTime=1730993030510; duration=0sec 2024-11-07T15:23:50,620 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:50,620 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:C 2024-11-07T15:23:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T15:23:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:50,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:50,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993090709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993090711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993090712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993090713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993090713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993090821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993090821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993090821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993090821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993090830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:50,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:50,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:50,940 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/6101f2f1ae72407db882e6929cb716d9 2024-11-07T15:23:50,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/d3b7c31940c34661943c600a41a7b8f5 is 50, key is test_row_0/B:col10/1730993030070/Put/seqid=0 2024-11-07T15:23:50,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742140_1316 (size=12301) 2024-11-07T15:23:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T15:23:51,005 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/a5bd3c23bc69487d95248bc9e769df0e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/a5bd3c23bc69487d95248bc9e769df0e 2024-11-07T15:23:51,012 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/B of b93a997d5d499fc85333e4dccefd8493 into a5bd3c23bc69487d95248bc9e769df0e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:51,012 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:51,012 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/B, priority=12, startTime=1730993030510; duration=0sec 2024-11-07T15:23:51,012 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:51,012 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:B 2024-11-07T15:23:51,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993091030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993091030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993091030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993091030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993091041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993091341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993091341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993091341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993091341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993091350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,358 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/d3b7c31940c34661943c600a41a7b8f5 2024-11-07T15:23:51,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/31e4d94e70ea4d5a922a02faefa249d8 is 50, key is test_row_0/C:col10/1730993030070/Put/seqid=0 2024-11-07T15:23:51,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742141_1317 (size=12301) 2024-11-07T15:23:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T15:23:51,769 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/31e4d94e70ea4d5a922a02faefa249d8 2024-11-07T15:23:51,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/6101f2f1ae72407db882e6929cb716d9 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/6101f2f1ae72407db882e6929cb716d9 2024-11-07T15:23:51,778 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/6101f2f1ae72407db882e6929cb716d9, entries=150, sequenceid=325, filesize=12.0 K 2024-11-07T15:23:51,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/d3b7c31940c34661943c600a41a7b8f5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/d3b7c31940c34661943c600a41a7b8f5 2024-11-07T15:23:51,783 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/d3b7c31940c34661943c600a41a7b8f5, entries=150, sequenceid=325, filesize=12.0 K 2024-11-07T15:23:51,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/31e4d94e70ea4d5a922a02faefa249d8 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/31e4d94e70ea4d5a922a02faefa249d8 2024-11-07T15:23:51,789 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/31e4d94e70ea4d5a922a02faefa249d8, entries=150, sequenceid=325, filesize=12.0 K 2024-11-07T15:23:51,789 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for b93a997d5d499fc85333e4dccefd8493 in 1270ms, sequenceid=325, compaction requested=false 2024-11-07T15:23:51,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:51,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:51,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-07T15:23:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-07T15:23:51,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-07T15:23:51,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4270 sec 2024-11-07T15:23:51,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.4310 sec 2024-11-07T15:23:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:51,853 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:51,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T15:23:51,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/7b5b3decf3be407d944492b7b6bd5cfd is 50, key is test_row_0/A:col10/1730993031853/Put/seqid=0 2024-11-07T15:23:51,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742142_1318 (size=12301) 2024-11-07T15:23:51,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993091871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993091872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993091872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993091874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993091877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993091980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993091980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993091981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993091981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:51,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993091991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:52,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993092190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993092191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:52,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993092191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993092191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:52,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993092200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:52,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/7b5b3decf3be407d944492b7b6bd5cfd 2024-11-07T15:23:52,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/4bafbd1da04d40879d0b4d35729d1b77 is 50, key is test_row_0/B:col10/1730993031853/Put/seqid=0 2024-11-07T15:23:52,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742143_1319 (size=12301) 2024-11-07T15:23:52,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993092500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:52,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993092501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993092500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993092501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T15:23:52,511 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:52,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-07T15:23:52,513 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T15:23:52,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993092511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:52,513 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:52,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T15:23:52,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T15:23:52,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:52,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:52,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:52,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:52,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:52,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:23:52,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/4bafbd1da04d40879d0b4d35729d1b77 2024-11-07T15:23:52,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/13cb8e549b224c68a24c9cdeb84b19cc is 50, key is test_row_0/C:col10/1730993031853/Put/seqid=0 2024-11-07T15:23:52,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742144_1320 (size=12301) 2024-11-07T15:23:52,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/13cb8e549b224c68a24c9cdeb84b19cc 2024-11-07T15:23:52,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/7b5b3decf3be407d944492b7b6bd5cfd as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/7b5b3decf3be407d944492b7b6bd5cfd 2024-11-07T15:23:52,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/7b5b3decf3be407d944492b7b6bd5cfd, entries=150, sequenceid=343, filesize=12.0 K 2024-11-07T15:23:52,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/4bafbd1da04d40879d0b4d35729d1b77 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/4bafbd1da04d40879d0b4d35729d1b77 2024-11-07T15:23:52,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/4bafbd1da04d40879d0b4d35729d1b77, entries=150, sequenceid=343, filesize=12.0 K 2024-11-07T15:23:52,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/13cb8e549b224c68a24c9cdeb84b19cc as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/13cb8e549b224c68a24c9cdeb84b19cc 2024-11-07T15:23:52,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/13cb8e549b224c68a24c9cdeb84b19cc, entries=150, sequenceid=343, filesize=12.0 K 2024-11-07T15:23:52,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for b93a997d5d499fc85333e4dccefd8493 in 854ms, sequenceid=343, compaction requested=true 2024-11-07T15:23:52,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:52,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:52,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:52,707 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:52,707 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:52,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:52,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:52,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:52,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:52,708 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:52,708 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/B is initiating minor compaction (all files) 2024-11-07T15:23:52,708 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/B in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:52,708 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/a5bd3c23bc69487d95248bc9e769df0e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/d3b7c31940c34661943c600a41a7b8f5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/4bafbd1da04d40879d0b4d35729d1b77] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=36.8 K 2024-11-07T15:23:52,708 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:52,708 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/A is initiating minor compaction (all files) 2024-11-07T15:23:52,708 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/A in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:52,709 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/c0dc3294e88d42c9a2b1885a21ef4c53, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/6101f2f1ae72407db882e6929cb716d9, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/7b5b3decf3be407d944492b7b6bd5cfd] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=36.8 K 2024-11-07T15:23:52,709 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting a5bd3c23bc69487d95248bc9e769df0e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1730993030038 2024-11-07T15:23:52,709 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0dc3294e88d42c9a2b1885a21ef4c53, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1730993030038 2024-11-07T15:23:52,709 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6101f2f1ae72407db882e6929cb716d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1730993030059 2024-11-07T15:23:52,709 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting d3b7c31940c34661943c600a41a7b8f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1730993030059 2024-11-07T15:23:52,710 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b5b3decf3be407d944492b7b6bd5cfd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1730993030711 2024-11-07T15:23:52,710 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bafbd1da04d40879d0b4d35729d1b77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1730993030711 2024-11-07T15:23:52,718 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#B#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:52,719 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/8fccbcbc64b6452d9162fe2ae8e7a32a is 50, key is test_row_0/B:col10/1730993031853/Put/seqid=0 2024-11-07T15:23:52,726 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#A#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:52,727 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/bfe719a3c9a14356bd302a8dfd7c44c9 is 50, key is test_row_0/A:col10/1730993031853/Put/seqid=0 2024-11-07T15:23:52,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742145_1321 (size=13153) 2024-11-07T15:23:52,739 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/8fccbcbc64b6452d9162fe2ae8e7a32a as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/8fccbcbc64b6452d9162fe2ae8e7a32a 2024-11-07T15:23:52,745 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/B of b93a997d5d499fc85333e4dccefd8493 into 8fccbcbc64b6452d9162fe2ae8e7a32a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:52,745 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:52,745 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/B, priority=13, startTime=1730993032707; duration=0sec 2024-11-07T15:23:52,746 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:52,746 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:B 2024-11-07T15:23:52,746 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:52,747 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:52,747 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/C is initiating minor compaction (all files) 2024-11-07T15:23:52,747 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/C in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:52,747 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/21f08385f85643ad8aacc373f0aded34, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/31e4d94e70ea4d5a922a02faefa249d8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/13cb8e549b224c68a24c9cdeb84b19cc] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=36.7 K 2024-11-07T15:23:52,747 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 21f08385f85643ad8aacc373f0aded34, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1730993030038 2024-11-07T15:23:52,747 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 31e4d94e70ea4d5a922a02faefa249d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1730993030059 2024-11-07T15:23:52,748 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 13cb8e549b224c68a24c9cdeb84b19cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1730993030711 2024-11-07T15:23:52,770 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#C#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:52,770 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/b5d5bd0b44e542008dd2846086635ed1 is 50, key is test_row_0/C:col10/1730993031853/Put/seqid=0 2024-11-07T15:23:52,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742146_1322 (size=13153) 2024-11-07T15:23:52,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742147_1323 (size=13119) 2024-11-07T15:23:52,816 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/b5d5bd0b44e542008dd2846086635ed1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/b5d5bd0b44e542008dd2846086635ed1 2024-11-07T15:23:52,816 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/bfe719a3c9a14356bd302a8dfd7c44c9 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/bfe719a3c9a14356bd302a8dfd7c44c9 2024-11-07T15:23:52,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T15:23:52,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:52,819 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T15:23:52,823 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/A of b93a997d5d499fc85333e4dccefd8493 into bfe719a3c9a14356bd302a8dfd7c44c9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:52,823 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:52,823 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/A, priority=13, startTime=1730993032707; duration=0sec 2024-11-07T15:23:52,823 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:52,823 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:A 2024-11-07T15:23:52,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/5ff2d95965494441a5557b8993675f94 is 50, key is test_row_0/A:col10/1730993031875/Put/seqid=0 2024-11-07T15:23:52,827 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/C of b93a997d5d499fc85333e4dccefd8493 into b5d5bd0b44e542008dd2846086635ed1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:52,827 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:52,827 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/C, priority=13, startTime=1730993032707; duration=0sec 2024-11-07T15:23:52,827 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:52,827 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:C 2024-11-07T15:23:52,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742148_1324 (size=12301) 2024-11-07T15:23:52,830 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/5ff2d95965494441a5557b8993675f94 2024-11-07T15:23:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T15:23:52,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/0e11f88e922c4f5ab6a55860c1ca2572 is 50, key is test_row_0/B:col10/1730993031875/Put/seqid=0 2024-11-07T15:23:52,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742149_1325 (size=12301) 2024-11-07T15:23:53,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:53,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:53,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993093022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993093023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993093023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993093025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993093025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993093131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993093131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993093131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993093131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993093131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T15:23:53,243 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/0e11f88e922c4f5ab6a55860c1ca2572 2024-11-07T15:23:53,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/a3db2119c91947b393ee22ffe47be51b is 50, key is test_row_0/C:col10/1730993031875/Put/seqid=0 2024-11-07T15:23:53,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742150_1326 (size=12301) 2024-11-07T15:23:53,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993093340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993093341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993093341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993093341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993093341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T15:23:53,653 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/a3db2119c91947b393ee22ffe47be51b 2024-11-07T15:23:53,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993093651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993093652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993093652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993093652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993093652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:53,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/5ff2d95965494441a5557b8993675f94 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/5ff2d95965494441a5557b8993675f94 2024-11-07T15:23:53,661 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/5ff2d95965494441a5557b8993675f94, entries=150, sequenceid=365, filesize=12.0 K 2024-11-07T15:23:53,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/0e11f88e922c4f5ab6a55860c1ca2572 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/0e11f88e922c4f5ab6a55860c1ca2572 2024-11-07T15:23:53,665 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/0e11f88e922c4f5ab6a55860c1ca2572, entries=150, sequenceid=365, filesize=12.0 K 2024-11-07T15:23:53,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/a3db2119c91947b393ee22ffe47be51b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/a3db2119c91947b393ee22ffe47be51b 2024-11-07T15:23:53,669 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/a3db2119c91947b393ee22ffe47be51b, entries=150, sequenceid=365, filesize=12.0 K 2024-11-07T15:23:53,670 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for b93a997d5d499fc85333e4dccefd8493 in 851ms, sequenceid=365, compaction requested=false 2024-11-07T15:23:53,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:53,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:53,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-07T15:23:53,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-07T15:23:53,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-07T15:23:53,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1580 sec 2024-11-07T15:23:53,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.1630 sec 2024-11-07T15:23:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:54,164 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:54,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-07T15:23:54,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/86dbbf56c10d418b87fa2880c3007c8b is 50, key is test_row_0/A:col10/1730993033024/Put/seqid=0 2024-11-07T15:23:54,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742151_1327 (size=12301) 2024-11-07T15:23:54,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/86dbbf56c10d418b87fa2880c3007c8b 2024-11-07T15:23:54,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/a5d87df58cb34415858b91a7815a4fda is 50, key is test_row_0/B:col10/1730993033024/Put/seqid=0 2024-11-07T15:23:54,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742152_1328 (size=12301) 2024-11-07T15:23:54,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993094201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993094201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993094202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993094202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993094202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993094311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993094311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993094311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993094311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993094311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993094521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993094521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993094521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993094521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993094521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/a5d87df58cb34415858b91a7815a4fda 2024-11-07T15:23:54,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/c128c19b9af642a2816e0c7162efa115 is 50, key is test_row_0/C:col10/1730993033024/Put/seqid=0 2024-11-07T15:23:54,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742153_1329 (size=12301) 2024-11-07T15:23:54,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/c128c19b9af642a2816e0c7162efa115 2024-11-07T15:23:54,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/86dbbf56c10d418b87fa2880c3007c8b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/86dbbf56c10d418b87fa2880c3007c8b 2024-11-07T15:23:54,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/86dbbf56c10d418b87fa2880c3007c8b, entries=150, sequenceid=387, filesize=12.0 K 2024-11-07T15:23:54,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/a5d87df58cb34415858b91a7815a4fda as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/a5d87df58cb34415858b91a7815a4fda 2024-11-07T15:23:54,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/a5d87df58cb34415858b91a7815a4fda, entries=150, sequenceid=387, filesize=12.0 K 2024-11-07T15:23:54,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/c128c19b9af642a2816e0c7162efa115 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/c128c19b9af642a2816e0c7162efa115 2024-11-07T15:23:54,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/c128c19b9af642a2816e0c7162efa115, entries=150, sequenceid=387, filesize=12.0 K 2024-11-07T15:23:54,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=100.63 KB/103050 for b93a997d5d499fc85333e4dccefd8493 in 462ms, sequenceid=387, compaction requested=true 2024-11-07T15:23:54,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:54,627 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:54,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:54,628 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:54,628 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/A is initiating minor compaction (all files) 2024-11-07T15:23:54,628 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/A in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:54,628 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/bfe719a3c9a14356bd302a8dfd7c44c9, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/5ff2d95965494441a5557b8993675f94, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/86dbbf56c10d418b87fa2880c3007c8b] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=36.9 K 2024-11-07T15:23:54,628 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfe719a3c9a14356bd302a8dfd7c44c9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1730993030711 2024-11-07T15:23:54,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:54,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:54,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:54,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:23:54,628 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:54,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:54,632 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ff2d95965494441a5557b8993675f94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1730993031869 2024-11-07T15:23:54,632 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86dbbf56c10d418b87fa2880c3007c8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1730993033024 2024-11-07T15:23:54,633 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:54,633 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/B is initiating minor compaction (all files) 2024-11-07T15:23:54,633 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/B in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:54,633 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/8fccbcbc64b6452d9162fe2ae8e7a32a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/0e11f88e922c4f5ab6a55860c1ca2572, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/a5d87df58cb34415858b91a7815a4fda] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=36.9 K 2024-11-07T15:23:54,633 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fccbcbc64b6452d9162fe2ae8e7a32a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1730993030711 2024-11-07T15:23:54,633 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e11f88e922c4f5ab6a55860c1ca2572, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1730993031869 2024-11-07T15:23:54,634 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting a5d87df58cb34415858b91a7815a4fda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1730993033024 2024-11-07T15:23:54,639 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#A#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:54,639 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/c8f0d4aaa2d64595bc254e109bc459cd is 50, key is test_row_0/A:col10/1730993033024/Put/seqid=0 2024-11-07T15:23:54,642 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#B#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:54,643 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/a80820683ddc4840841b84b45a705159 is 50, key is test_row_0/B:col10/1730993033024/Put/seqid=0 2024-11-07T15:23:54,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742154_1330 (size=13255) 2024-11-07T15:23:54,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742155_1331 (size=13255) 2024-11-07T15:23:54,659 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/a80820683ddc4840841b84b45a705159 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/a80820683ddc4840841b84b45a705159 2024-11-07T15:23:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T15:23:54,661 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:54,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:23:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-07T15:23:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T15:23:54,664 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:23:54,665 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:23:54,665 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:23:54,665 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/B of b93a997d5d499fc85333e4dccefd8493 into a80820683ddc4840841b84b45a705159(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:54,665 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:54,666 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/B, priority=13, startTime=1730993034628; duration=0sec 2024-11-07T15:23:54,666 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:23:54,666 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:B 2024-11-07T15:23:54,666 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:23:54,668 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:23:54,668 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): b93a997d5d499fc85333e4dccefd8493/C is initiating minor compaction (all files) 2024-11-07T15:23:54,668 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b93a997d5d499fc85333e4dccefd8493/C in TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:54,668 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/b5d5bd0b44e542008dd2846086635ed1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/a3db2119c91947b393ee22ffe47be51b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/c128c19b9af642a2816e0c7162efa115] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp, totalSize=36.8 K 2024-11-07T15:23:54,669 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting b5d5bd0b44e542008dd2846086635ed1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1730993030711 2024-11-07T15:23:54,669 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting a3db2119c91947b393ee22ffe47be51b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1730993031869 2024-11-07T15:23:54,669 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting c128c19b9af642a2816e0c7162efa115, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1730993033024 2024-11-07T15:23:54,674 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b93a997d5d499fc85333e4dccefd8493#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:23:54,674 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/25d0b2c138964038a27a224535a1ac1e is 50, key is test_row_0/C:col10/1730993033024/Put/seqid=0 2024-11-07T15:23:54,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742156_1332 (size=13221) 2024-11-07T15:23:54,686 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/25d0b2c138964038a27a224535a1ac1e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/25d0b2c138964038a27a224535a1ac1e 2024-11-07T15:23:54,691 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/C of b93a997d5d499fc85333e4dccefd8493 into 25d0b2c138964038a27a224535a1ac1e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:54,691 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:54,691 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/C, priority=13, startTime=1730993034628; duration=0sec 2024-11-07T15:23:54,691 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:54,691 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:C 2024-11-07T15:23:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T15:23:54,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T15:23:54,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:54,818 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T15:23:54,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/d2f543d181e949969af511b09cd19234 is 50, key is test_row_0/A:col10/1730993034201/Put/seqid=0 2024-11-07T15:23:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:54,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. as already flushing 2024-11-07T15:23:54,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742157_1333 (size=12301) 2024-11-07T15:23:54,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993094844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993094844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993094846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993094847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993094848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993094951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993094951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993094961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993094961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993094970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T15:23:55,058 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/c8f0d4aaa2d64595bc254e109bc459cd as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/c8f0d4aaa2d64595bc254e109bc459cd 2024-11-07T15:23:55,065 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b93a997d5d499fc85333e4dccefd8493/A of b93a997d5d499fc85333e4dccefd8493 into c8f0d4aaa2d64595bc254e109bc459cd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:23:55,065 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:55,065 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., storeName=b93a997d5d499fc85333e4dccefd8493/A, priority=13, startTime=1730993034626; duration=0sec 2024-11-07T15:23:55,065 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:55,065 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:A 2024-11-07T15:23:55,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993095160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993095160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:55,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993095171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:55,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:55,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993095171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:55,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993095181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:55,245 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/d2f543d181e949969af511b09cd19234 2024-11-07T15:23:55,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/28a2de5c6bea4dc8b05d2a79cfe26b44 is 50, key is test_row_0/B:col10/1730993034201/Put/seqid=0 2024-11-07T15:23:55,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742158_1334 (size=12301) 2024-11-07T15:23:55,268 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/28a2de5c6bea4dc8b05d2a79cfe26b44 2024-11-07T15:23:55,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/3a5161060d644e4fa667a1e2c90c1cf5 is 50, key is test_row_0/C:col10/1730993034201/Put/seqid=0 2024-11-07T15:23:55,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742159_1335 (size=12301) 2024-11-07T15:23:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T15:23:55,389 INFO [Thread-1162 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1162. 2024-11-07T15:23:55,389 INFO [Thread-1154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1154. 2024-11-07T15:23:55,389 DEBUG [Thread-1154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:55,389 DEBUG [Thread-1162 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:55,389 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,389 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,389 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,389 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:55,389 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:55,390 INFO [Thread-1156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1156. 2024-11-07T15:23:55,390 DEBUG [Thread-1156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:55,390 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,390 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:55,392 INFO [Thread-1160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1160. 2024-11-07T15:23:55,392 DEBUG [Thread-1160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:55,392 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,392 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,392 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:55,393 INFO [Thread-1158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1158. 2024-11-07T15:23:55,393 DEBUG [Thread-1158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:55,393 DEBUG [Thread-1158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,393 DEBUG [Thread-1158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:55,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:55,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43160 deadline: 1730993095471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43138 deadline: 1730993095471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:55,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:55,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43162 deadline: 1730993095480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43118 deadline: 1730993095480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:55,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:23:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43096 deadline: 1730993095491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:23:55,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:23:55,682 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/3a5161060d644e4fa667a1e2c90c1cf5 2024-11-07T15:23:55,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/d2f543d181e949969af511b09cd19234 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/d2f543d181e949969af511b09cd19234 2024-11-07T15:23:55,690 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/d2f543d181e949969af511b09cd19234, entries=150, sequenceid=407, filesize=12.0 K 2024-11-07T15:23:55,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/28a2de5c6bea4dc8b05d2a79cfe26b44 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/28a2de5c6bea4dc8b05d2a79cfe26b44 2024-11-07T15:23:55,699 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/28a2de5c6bea4dc8b05d2a79cfe26b44, entries=150, sequenceid=407, filesize=12.0 K 2024-11-07T15:23:55,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/3a5161060d644e4fa667a1e2c90c1cf5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/3a5161060d644e4fa667a1e2c90c1cf5 2024-11-07T15:23:55,703 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/3a5161060d644e4fa667a1e2c90c1cf5, entries=150, sequenceid=407, filesize=12.0 K 2024-11-07T15:23:55,704 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for b93a997d5d499fc85333e4dccefd8493 in 886ms, sequenceid=407, compaction requested=false 2024-11-07T15:23:55,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:55,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:55,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-07T15:23:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-07T15:23:55,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-07T15:23:55,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0400 sec 2024-11-07T15:23:55,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.0460 sec 2024-11-07T15:23:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T15:23:55,801 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:23:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:55,982 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:23:55,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T15:23:55,982 INFO [Thread-1143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1143. 2024-11-07T15:23:55,982 INFO [Thread-1149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1149. 2024-11-07T15:23:55,982 DEBUG [Thread-1143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:55,982 DEBUG [Thread-1149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:55,982 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,982 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,982 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,982 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,982 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:55,982 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:55,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/fa0bcd0c30b44727bc57f0e8a7cf33cc is 50, key is test_row_0/A:col10/1730993034845/Put/seqid=0 2024-11-07T15:23:55,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742160_1336 (size=12301) 2024-11-07T15:23:55,992 INFO [Thread-1147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1147. 2024-11-07T15:23:55,992 INFO [Thread-1151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1151. 2024-11-07T15:23:55,992 DEBUG [Thread-1147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:55,992 DEBUG [Thread-1151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:55,992 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,992 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,992 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,992 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:55,992 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:55,992 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:56,002 INFO [Thread-1145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1145. 2024-11-07T15:23:56,002 DEBUG [Thread-1145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:23:56,003 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:56,003 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:56,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:56,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T15:23:56,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-07T15:23:56,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-07T15:23:56,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-07T15:23:56,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-07T15:23:56,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-07T15:23:56,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T15:23:56,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3631 2024-11-07T15:23:56,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3503 2024-11-07T15:23:56,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3465 2024-11-07T15:23:56,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3630 2024-11-07T15:23:56,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3506 2024-11-07T15:23:56,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T15:23:56,003 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T15:23:56,003 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:23:56,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:56,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:23:56,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:23:56,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T15:23:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T15:23:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T15:23:56,007 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993036006"}]},"ts":"1730993036006"} 2024-11-07T15:23:56,008 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T15:23:56,008 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T15:23:56,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:23:56,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b93a997d5d499fc85333e4dccefd8493, UNASSIGN}] 2024-11-07T15:23:56,010 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b93a997d5d499fc85333e4dccefd8493, UNASSIGN 2024-11-07T15:23:56,010 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=b93a997d5d499fc85333e4dccefd8493, regionState=CLOSING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:23:56,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b93a997d5d499fc85333e4dccefd8493, UNASSIGN because future has completed 2024-11-07T15:23:56,012 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:23:56,012 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:23:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T15:23:56,164 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:56,164 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:23:56,164 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing b93a997d5d499fc85333e4dccefd8493, disabling compactions & flushes 2024-11-07T15:23:56,164 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T15:23:56,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/fa0bcd0c30b44727bc57f0e8a7cf33cc 2024-11-07T15:23:56,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/8aaf9e3796444dc39d6cde0f74062b7a is 50, key is test_row_0/B:col10/1730993034845/Put/seqid=0 2024-11-07T15:23:56,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742161_1337 (size=12301) 2024-11-07T15:23:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T15:23:56,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/8aaf9e3796444dc39d6cde0f74062b7a 2024-11-07T15:23:56,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/092292199ce54c049fda6f28141594b5 is 50, key is test_row_0/C:col10/1730993034845/Put/seqid=0 2024-11-07T15:23:56,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742162_1338 (size=12301) 2024-11-07T15:23:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T15:23:57,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/092292199ce54c049fda6f28141594b5 2024-11-07T15:23:57,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/fa0bcd0c30b44727bc57f0e8a7cf33cc as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/fa0bcd0c30b44727bc57f0e8a7cf33cc 2024-11-07T15:23:57,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/fa0bcd0c30b44727bc57f0e8a7cf33cc, entries=150, sequenceid=428, filesize=12.0 K 2024-11-07T15:23:57,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/8aaf9e3796444dc39d6cde0f74062b7a as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/8aaf9e3796444dc39d6cde0f74062b7a 2024-11-07T15:23:57,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/8aaf9e3796444dc39d6cde0f74062b7a, entries=150, sequenceid=428, filesize=12.0 K 2024-11-07T15:23:57,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/092292199ce54c049fda6f28141594b5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/092292199ce54c049fda6f28141594b5 2024-11-07T15:23:57,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/092292199ce54c049fda6f28141594b5, entries=150, sequenceid=428, filesize=12.0 K 2024-11-07T15:23:57,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=20.13 KB/20610 for b93a997d5d499fc85333e4dccefd8493 in 1241ms, sequenceid=428, compaction requested=true 2024-11-07T15:23:57,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b93a997d5d499fc85333e4dccefd8493: 2024-11-07T15:23:57,223 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:57,223 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:57,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:57,223 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. after waiting 0 ms 2024-11-07T15:23:57,223 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:57,223 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. because compaction request was cancelled 2024-11-07T15:23:57,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:57,223 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:A 2024-11-07T15:23:57,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:23:57,223 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing b93a997d5d499fc85333e4dccefd8493 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-07T15:23:57,223 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. because compaction request was cancelled 2024-11-07T15:23:57,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:57,223 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:B 2024-11-07T15:23:57,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b93a997d5d499fc85333e4dccefd8493:C, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:23:57,223 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. because compaction request was cancelled 2024-11-07T15:23:57,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:23:57,223 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b93a997d5d499fc85333e4dccefd8493:C 2024-11-07T15:23:57,226 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/3b63c0074fff40efb350e46063e85e4d is 50, key is test_row_0/A:col10/1730993036001/Put/seqid=0 2024-11-07T15:23:57,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742163_1339 (size=9857) 2024-11-07T15:23:57,632 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/3b63c0074fff40efb350e46063e85e4d 2024-11-07T15:23:57,637 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/8f117289977c43ebbb1037ce8d1e07a0 is 50, key is test_row_0/B:col10/1730993036001/Put/seqid=0 2024-11-07T15:23:57,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742164_1340 (size=9857) 2024-11-07T15:23:58,041 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/8f117289977c43ebbb1037ce8d1e07a0 2024-11-07T15:23:58,047 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/409e519c5bb14418a0698a00a2aecbbf is 50, key is test_row_0/C:col10/1730993036001/Put/seqid=0 2024-11-07T15:23:58,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742165_1341 (size=9857) 2024-11-07T15:23:58,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T15:23:58,456 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/409e519c5bb14418a0698a00a2aecbbf 2024-11-07T15:23:58,460 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/A/3b63c0074fff40efb350e46063e85e4d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/3b63c0074fff40efb350e46063e85e4d 2024-11-07T15:23:58,462 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/3b63c0074fff40efb350e46063e85e4d, entries=100, sequenceid=434, filesize=9.6 K 2024-11-07T15:23:58,463 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/B/8f117289977c43ebbb1037ce8d1e07a0 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/8f117289977c43ebbb1037ce8d1e07a0 2024-11-07T15:23:58,467 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/8f117289977c43ebbb1037ce8d1e07a0, entries=100, sequenceid=434, filesize=9.6 K 2024-11-07T15:23:58,468 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/.tmp/C/409e519c5bb14418a0698a00a2aecbbf as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/409e519c5bb14418a0698a00a2aecbbf 2024-11-07T15:23:58,471 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/409e519c5bb14418a0698a00a2aecbbf, entries=100, sequenceid=434, filesize=9.6 K 2024-11-07T15:23:58,472 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for b93a997d5d499fc85333e4dccefd8493 in 1248ms, sequenceid=434, compaction requested=true 2024-11-07T15:23:58,472 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/a90e47b3e42a453e8462a11ef1abb11e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/1a7e62004b404d2ea2a70398cedfa260, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/3f543971b5674376bfabfd66500a56d8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/6adb6acd7df648d3a765e2af882fcab6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/48c6e8de8a0c4b02ad12773c8e3f4eb1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/a2519b2cf64d4655bbd53422749540c0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/afb4ad3123f3479a806e3088886f0c47, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/5dd6bb70a080496886f382fff38ed63b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/af80c11ec0914e388f9b375c1074dcd8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/99d228a2303c4b6d8cd22d428916f62d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/c543cc96486e48b38711e2e1794a19d5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/ce6302b1978447e3bb77bd16c6c260f4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/ec8527c2a0454d31a1dc61006e88194b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/48c00bf7acc54a79ae86eed7aa812252, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/65bd3751cfef4d69a8113441557a8f82, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/346b3be6483845d1bf654ad7d6ab86c3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/06049217041643fab8fc1cb259f6dd37, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/df974b9a66e04c40a1496784c933bfc5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/7a611ac2e8374795af3795290659fe47, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/29510235f92f4cb8a0578384e2905309, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/d3b8372fd18f4cbf8080ddcb66fb94bd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/c0dc3294e88d42c9a2b1885a21ef4c53, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/631b0b6c8aee416ea22e549f7778f93d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/6101f2f1ae72407db882e6929cb716d9, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/bfe719a3c9a14356bd302a8dfd7c44c9, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/7b5b3decf3be407d944492b7b6bd5cfd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/5ff2d95965494441a5557b8993675f94, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/86dbbf56c10d418b87fa2880c3007c8b] to archive 2024-11-07T15:23:58,473 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:23:58,474 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/a90e47b3e42a453e8462a11ef1abb11e to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/a90e47b3e42a453e8462a11ef1abb11e 2024-11-07T15:23:58,475 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/1a7e62004b404d2ea2a70398cedfa260 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/1a7e62004b404d2ea2a70398cedfa260 2024-11-07T15:23:58,476 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/3f543971b5674376bfabfd66500a56d8 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/3f543971b5674376bfabfd66500a56d8 2024-11-07T15:23:58,477 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/6adb6acd7df648d3a765e2af882fcab6 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/6adb6acd7df648d3a765e2af882fcab6 2024-11-07T15:23:58,478 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/48c6e8de8a0c4b02ad12773c8e3f4eb1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/48c6e8de8a0c4b02ad12773c8e3f4eb1 2024-11-07T15:23:58,479 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/a2519b2cf64d4655bbd53422749540c0 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/a2519b2cf64d4655bbd53422749540c0 2024-11-07T15:23:58,480 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/afb4ad3123f3479a806e3088886f0c47 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/afb4ad3123f3479a806e3088886f0c47 2024-11-07T15:23:58,481 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/5dd6bb70a080496886f382fff38ed63b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/5dd6bb70a080496886f382fff38ed63b 2024-11-07T15:23:58,482 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/af80c11ec0914e388f9b375c1074dcd8 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/af80c11ec0914e388f9b375c1074dcd8 2024-11-07T15:23:58,482 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/99d228a2303c4b6d8cd22d428916f62d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/99d228a2303c4b6d8cd22d428916f62d 2024-11-07T15:23:58,483 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/c543cc96486e48b38711e2e1794a19d5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/c543cc96486e48b38711e2e1794a19d5 2024-11-07T15:23:58,484 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/ce6302b1978447e3bb77bd16c6c260f4 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/ce6302b1978447e3bb77bd16c6c260f4 2024-11-07T15:23:58,485 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/ec8527c2a0454d31a1dc61006e88194b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/ec8527c2a0454d31a1dc61006e88194b 2024-11-07T15:23:58,485 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/48c00bf7acc54a79ae86eed7aa812252 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/48c00bf7acc54a79ae86eed7aa812252 2024-11-07T15:23:58,486 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/65bd3751cfef4d69a8113441557a8f82 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/65bd3751cfef4d69a8113441557a8f82 2024-11-07T15:23:58,487 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/346b3be6483845d1bf654ad7d6ab86c3 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/346b3be6483845d1bf654ad7d6ab86c3 2024-11-07T15:23:58,487 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/06049217041643fab8fc1cb259f6dd37 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/06049217041643fab8fc1cb259f6dd37 2024-11-07T15:23:58,488 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/df974b9a66e04c40a1496784c933bfc5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/df974b9a66e04c40a1496784c933bfc5 2024-11-07T15:23:58,489 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/7a611ac2e8374795af3795290659fe47 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/7a611ac2e8374795af3795290659fe47 2024-11-07T15:23:58,489 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/29510235f92f4cb8a0578384e2905309 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/29510235f92f4cb8a0578384e2905309 2024-11-07T15:23:58,490 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/d3b8372fd18f4cbf8080ddcb66fb94bd to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/d3b8372fd18f4cbf8080ddcb66fb94bd 2024-11-07T15:23:58,491 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/c0dc3294e88d42c9a2b1885a21ef4c53 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/c0dc3294e88d42c9a2b1885a21ef4c53 2024-11-07T15:23:58,492 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/631b0b6c8aee416ea22e549f7778f93d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/631b0b6c8aee416ea22e549f7778f93d 2024-11-07T15:23:58,492 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/6101f2f1ae72407db882e6929cb716d9 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/6101f2f1ae72407db882e6929cb716d9 2024-11-07T15:23:58,493 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/bfe719a3c9a14356bd302a8dfd7c44c9 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/bfe719a3c9a14356bd302a8dfd7c44c9 2024-11-07T15:23:58,494 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/7b5b3decf3be407d944492b7b6bd5cfd to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/7b5b3decf3be407d944492b7b6bd5cfd 2024-11-07T15:23:58,494 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/5ff2d95965494441a5557b8993675f94 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/5ff2d95965494441a5557b8993675f94 2024-11-07T15:23:58,495 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/86dbbf56c10d418b87fa2880c3007c8b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/86dbbf56c10d418b87fa2880c3007c8b 2024-11-07T15:23:58,496 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/706e7d1b408a483aaf944888c0eb019a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/f39d30afb1ca4d01b3f08c6fc20d0b17, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/097c7cdae77d4e7c850f5fcd6a1f08ba, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/9c9caec1aa544459b654b892313db501, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/e4d6ed71e5394db39067bcf5018abc46, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/7954d864022443c3ae14ead53a57579d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/d4676a05163d4c04a4961d579f8fa59a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/16263a8c967d449fbacb68d195621124, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/30cc4f766dee4aa496cb56a320e9529c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/9d24b296821344b5b5ee61c799938911, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/860053da02cc40d48b98cff1b3884f02, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/6c0b02b5188c49f48ebb7c91c0f2dd87, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/5e68910146ee444d885f102cf5871e75, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/82351bb049a34a529ec35d9741d19ebc, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/5c0c7f7a6f254338a8e872af8f637dde, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/209dc81ec1a84562b4fc5a378aa018b6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/076c0a8686b64c359093737c275aa975, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/57e8f80db5b040489af5a0b24b3796d1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/93334fe4bcef4e908dd9643ff95c4c1c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/367fd6b2859247a9b6e16e5911768ac4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/3710a38e222a46c5a41d36fe5795ee70, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/a5bd3c23bc69487d95248bc9e769df0e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/4c48f56db1eb449389af6c377687bcb4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/d3b7c31940c34661943c600a41a7b8f5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/8fccbcbc64b6452d9162fe2ae8e7a32a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/4bafbd1da04d40879d0b4d35729d1b77, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/0e11f88e922c4f5ab6a55860c1ca2572, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/a5d87df58cb34415858b91a7815a4fda] to archive 2024-11-07T15:23:58,497 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:23:58,498 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/706e7d1b408a483aaf944888c0eb019a to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/706e7d1b408a483aaf944888c0eb019a 2024-11-07T15:23:58,499 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/f39d30afb1ca4d01b3f08c6fc20d0b17 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/f39d30afb1ca4d01b3f08c6fc20d0b17 2024-11-07T15:23:58,500 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/097c7cdae77d4e7c850f5fcd6a1f08ba to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/097c7cdae77d4e7c850f5fcd6a1f08ba 2024-11-07T15:23:58,501 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/9c9caec1aa544459b654b892313db501 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/9c9caec1aa544459b654b892313db501 2024-11-07T15:23:58,502 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/e4d6ed71e5394db39067bcf5018abc46 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/e4d6ed71e5394db39067bcf5018abc46 2024-11-07T15:23:58,503 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/7954d864022443c3ae14ead53a57579d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/7954d864022443c3ae14ead53a57579d 2024-11-07T15:23:58,504 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/d4676a05163d4c04a4961d579f8fa59a to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/d4676a05163d4c04a4961d579f8fa59a 2024-11-07T15:23:58,505 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/16263a8c967d449fbacb68d195621124 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/16263a8c967d449fbacb68d195621124 2024-11-07T15:23:58,506 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/30cc4f766dee4aa496cb56a320e9529c to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/30cc4f766dee4aa496cb56a320e9529c 2024-11-07T15:23:58,507 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/9d24b296821344b5b5ee61c799938911 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/9d24b296821344b5b5ee61c799938911 2024-11-07T15:23:58,507 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/860053da02cc40d48b98cff1b3884f02 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/860053da02cc40d48b98cff1b3884f02 2024-11-07T15:23:58,508 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/6c0b02b5188c49f48ebb7c91c0f2dd87 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/6c0b02b5188c49f48ebb7c91c0f2dd87 2024-11-07T15:23:58,509 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/5e68910146ee444d885f102cf5871e75 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/5e68910146ee444d885f102cf5871e75 2024-11-07T15:23:58,509 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/82351bb049a34a529ec35d9741d19ebc to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/82351bb049a34a529ec35d9741d19ebc 2024-11-07T15:23:58,510 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/5c0c7f7a6f254338a8e872af8f637dde to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/5c0c7f7a6f254338a8e872af8f637dde 2024-11-07T15:23:58,511 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/209dc81ec1a84562b4fc5a378aa018b6 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/209dc81ec1a84562b4fc5a378aa018b6 2024-11-07T15:23:58,511 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/076c0a8686b64c359093737c275aa975 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/076c0a8686b64c359093737c275aa975 2024-11-07T15:23:58,512 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/57e8f80db5b040489af5a0b24b3796d1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/57e8f80db5b040489af5a0b24b3796d1 2024-11-07T15:23:58,513 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/93334fe4bcef4e908dd9643ff95c4c1c to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/93334fe4bcef4e908dd9643ff95c4c1c 2024-11-07T15:23:58,513 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/367fd6b2859247a9b6e16e5911768ac4 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/367fd6b2859247a9b6e16e5911768ac4 2024-11-07T15:23:58,514 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/3710a38e222a46c5a41d36fe5795ee70 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/3710a38e222a46c5a41d36fe5795ee70 2024-11-07T15:23:58,515 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/a5bd3c23bc69487d95248bc9e769df0e to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/a5bd3c23bc69487d95248bc9e769df0e 2024-11-07T15:23:58,516 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/4c48f56db1eb449389af6c377687bcb4 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/4c48f56db1eb449389af6c377687bcb4 2024-11-07T15:23:58,516 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/d3b7c31940c34661943c600a41a7b8f5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/d3b7c31940c34661943c600a41a7b8f5 2024-11-07T15:23:58,517 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/8fccbcbc64b6452d9162fe2ae8e7a32a to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/8fccbcbc64b6452d9162fe2ae8e7a32a 2024-11-07T15:23:58,518 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/4bafbd1da04d40879d0b4d35729d1b77 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/4bafbd1da04d40879d0b4d35729d1b77 2024-11-07T15:23:58,518 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/0e11f88e922c4f5ab6a55860c1ca2572 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/0e11f88e922c4f5ab6a55860c1ca2572 2024-11-07T15:23:58,519 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/a5d87df58cb34415858b91a7815a4fda to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/a5d87df58cb34415858b91a7815a4fda 2024-11-07T15:23:58,520 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/279c8e02a2144ff89820ae11ff3eb8da, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/7c59da3bb53044359076398a223f89ad, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/c42044e081d24b098eab7400cd630a38, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/3e808f1410cc4ad48d9d2d24d144a8cb, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/98c2e12a72c04862b6da9451063b5b4a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/5f7f4b19021f4a7ab5f7b4d830a91e1b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/6ecbe067912344c4bc197a36e91c6763, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/10e329d0b6414e49b7af99af272ff504, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/92ce31d7ebcb4205a6a77cfce54bad3f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/012686647fd242cda005e8bb44b1667f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/b2b856d1e4744a78bba999e6cb748892, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/d5ca8dbea7324ac188d00c0b8b62d540, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/6ee4f3dc62cc414bb192aef81053cdad, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/b104ed28eb45484c92e521a14cef0ef0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/1279435b4d4e40b79fa3ff6acbb844ef, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/f58cc622cc0e44509c2ce1ebe7ccef1b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/bf71b7ce593142c5ab64a83effb9e788, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/4a0a25bb3be44b17866daaee87c7b8b2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/0db2dc769be54e2c9b5a13224dfef4a4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/647435839ffb45d09464d011bbe5f61e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/21f08385f85643ad8aacc373f0aded34, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/8c9ddcb57bbb41b884e2f5989cb56d24, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/31e4d94e70ea4d5a922a02faefa249d8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/b5d5bd0b44e542008dd2846086635ed1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/13cb8e549b224c68a24c9cdeb84b19cc, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/a3db2119c91947b393ee22ffe47be51b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/c128c19b9af642a2816e0c7162efa115] to archive 2024-11-07T15:23:58,521 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:23:58,522 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/279c8e02a2144ff89820ae11ff3eb8da to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/279c8e02a2144ff89820ae11ff3eb8da 2024-11-07T15:23:58,522 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/7c59da3bb53044359076398a223f89ad to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/7c59da3bb53044359076398a223f89ad 2024-11-07T15:23:58,523 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/c42044e081d24b098eab7400cd630a38 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/c42044e081d24b098eab7400cd630a38 2024-11-07T15:23:58,524 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/3e808f1410cc4ad48d9d2d24d144a8cb to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/3e808f1410cc4ad48d9d2d24d144a8cb 2024-11-07T15:23:58,524 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/98c2e12a72c04862b6da9451063b5b4a to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/98c2e12a72c04862b6da9451063b5b4a 2024-11-07T15:23:58,525 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/5f7f4b19021f4a7ab5f7b4d830a91e1b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/5f7f4b19021f4a7ab5f7b4d830a91e1b 2024-11-07T15:23:58,526 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/6ecbe067912344c4bc197a36e91c6763 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/6ecbe067912344c4bc197a36e91c6763 2024-11-07T15:23:58,527 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/10e329d0b6414e49b7af99af272ff504 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/10e329d0b6414e49b7af99af272ff504 2024-11-07T15:23:58,527 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/92ce31d7ebcb4205a6a77cfce54bad3f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/92ce31d7ebcb4205a6a77cfce54bad3f 2024-11-07T15:23:58,528 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/012686647fd242cda005e8bb44b1667f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/012686647fd242cda005e8bb44b1667f 2024-11-07T15:23:58,529 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/b2b856d1e4744a78bba999e6cb748892 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/b2b856d1e4744a78bba999e6cb748892 2024-11-07T15:23:58,530 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/d5ca8dbea7324ac188d00c0b8b62d540 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/d5ca8dbea7324ac188d00c0b8b62d540 2024-11-07T15:23:58,531 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/6ee4f3dc62cc414bb192aef81053cdad to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/6ee4f3dc62cc414bb192aef81053cdad 2024-11-07T15:23:58,531 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/b104ed28eb45484c92e521a14cef0ef0 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/b104ed28eb45484c92e521a14cef0ef0 2024-11-07T15:23:58,532 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/1279435b4d4e40b79fa3ff6acbb844ef to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/1279435b4d4e40b79fa3ff6acbb844ef 2024-11-07T15:23:58,533 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/f58cc622cc0e44509c2ce1ebe7ccef1b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/f58cc622cc0e44509c2ce1ebe7ccef1b 2024-11-07T15:23:58,534 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/bf71b7ce593142c5ab64a83effb9e788 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/bf71b7ce593142c5ab64a83effb9e788 2024-11-07T15:23:58,535 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/4a0a25bb3be44b17866daaee87c7b8b2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/4a0a25bb3be44b17866daaee87c7b8b2 2024-11-07T15:23:58,535 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/0db2dc769be54e2c9b5a13224dfef4a4 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/0db2dc769be54e2c9b5a13224dfef4a4 2024-11-07T15:23:58,536 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/647435839ffb45d09464d011bbe5f61e to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/647435839ffb45d09464d011bbe5f61e 2024-11-07T15:23:58,537 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/21f08385f85643ad8aacc373f0aded34 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/21f08385f85643ad8aacc373f0aded34 2024-11-07T15:23:58,538 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/8c9ddcb57bbb41b884e2f5989cb56d24 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/8c9ddcb57bbb41b884e2f5989cb56d24 2024-11-07T15:23:58,538 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/31e4d94e70ea4d5a922a02faefa249d8 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/31e4d94e70ea4d5a922a02faefa249d8 2024-11-07T15:23:58,539 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/b5d5bd0b44e542008dd2846086635ed1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/b5d5bd0b44e542008dd2846086635ed1 2024-11-07T15:23:58,540 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/13cb8e549b224c68a24c9cdeb84b19cc to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/13cb8e549b224c68a24c9cdeb84b19cc 2024-11-07T15:23:58,541 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/a3db2119c91947b393ee22ffe47be51b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/a3db2119c91947b393ee22ffe47be51b 2024-11-07T15:23:58,541 DEBUG [StoreCloser-TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/c128c19b9af642a2816e0c7162efa115 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/c128c19b9af642a2816e0c7162efa115 2024-11-07T15:23:58,546 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/recovered.edits/437.seqid, newMaxSeqId=437, maxSeqId=1 2024-11-07T15:23:58,546 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. 2024-11-07T15:23:58,546 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for b93a997d5d499fc85333e4dccefd8493: Waiting for close lock at 1730993036164Running coprocessor pre-close hooks at 1730993036164Disabling compacts and flushes for region at 1730993036164Disabling writes for close at 1730993037223 (+1059 ms)Obtaining lock to block concurrent updates at 1730993037223Preparing flush snapshotting stores in b93a997d5d499fc85333e4dccefd8493 at 1730993037223Finished memstore snapshotting TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1730993037223Flushing stores of TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493. at 1730993037224 (+1 ms)Flushing b93a997d5d499fc85333e4dccefd8493/A: creating writer at 1730993037224Flushing b93a997d5d499fc85333e4dccefd8493/A: appending metadata at 1730993037226 (+2 ms)Flushing b93a997d5d499fc85333e4dccefd8493/A: closing flushed file at 1730993037226Flushing b93a997d5d499fc85333e4dccefd8493/B: creating writer at 1730993037635 (+409 ms)Flushing b93a997d5d499fc85333e4dccefd8493/B: appending metadata at 1730993037637 (+2 ms)Flushing b93a997d5d499fc85333e4dccefd8493/B: closing flushed file at 1730993037637Flushing b93a997d5d499fc85333e4dccefd8493/C: creating writer at 1730993038044 (+407 ms)Flushing b93a997d5d499fc85333e4dccefd8493/C: appending metadata at 1730993038046 (+2 ms)Flushing b93a997d5d499fc85333e4dccefd8493/C: closing flushed file at 1730993038046Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56ef4ec9: reopening flushed file at 1730993038459 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24fdbdd2: reopening flushed file at 1730993038463 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f90e260: reopening flushed file at 1730993038467 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for b93a997d5d499fc85333e4dccefd8493 in 1248ms, sequenceid=434, compaction requested=true at 1730993038472 (+5 ms)Writing region close event to WAL at 1730993038543 (+71 ms)Running coprocessor post-close hooks at 1730993038546 (+3 ms)Closed at 1730993038546 2024-11-07T15:23:58,548 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:23:58,548 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=b93a997d5d499fc85333e4dccefd8493, regionState=CLOSED 2024-11-07T15:23:58,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:23:58,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-07T15:23:58,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure b93a997d5d499fc85333e4dccefd8493, server=7b3e4e8e12ef,33849,1730992951906 in 2.5380 sec 2024-11-07T15:23:58,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-07T15:23:58,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b93a997d5d499fc85333e4dccefd8493, UNASSIGN in 2.5420 sec 2024-11-07T15:23:58,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-07T15:23:58,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5450 sec 2024-11-07T15:23:58,555 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993038555"}]},"ts":"1730993038555"} 2024-11-07T15:23:58,556 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T15:23:58,556 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T15:23:58,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5530 sec 2024-11-07T15:24:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T15:24:00,161 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:00,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T15:24:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:00,164 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T15:24:00,164 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:00,167 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:24:00,169 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/recovered.edits] 2024-11-07T15:24:00,172 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/3b63c0074fff40efb350e46063e85e4d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/3b63c0074fff40efb350e46063e85e4d 2024-11-07T15:24:00,173 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/c8f0d4aaa2d64595bc254e109bc459cd to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/c8f0d4aaa2d64595bc254e109bc459cd 2024-11-07T15:24:00,174 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/d2f543d181e949969af511b09cd19234 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/d2f543d181e949969af511b09cd19234 2024-11-07T15:24:00,175 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/fa0bcd0c30b44727bc57f0e8a7cf33cc to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/A/fa0bcd0c30b44727bc57f0e8a7cf33cc 2024-11-07T15:24:00,176 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/28a2de5c6bea4dc8b05d2a79cfe26b44 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/28a2de5c6bea4dc8b05d2a79cfe26b44 2024-11-07T15:24:00,177 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/8aaf9e3796444dc39d6cde0f74062b7a to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/8aaf9e3796444dc39d6cde0f74062b7a 2024-11-07T15:24:00,178 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/8f117289977c43ebbb1037ce8d1e07a0 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/8f117289977c43ebbb1037ce8d1e07a0 2024-11-07T15:24:00,179 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/a80820683ddc4840841b84b45a705159 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/B/a80820683ddc4840841b84b45a705159 2024-11-07T15:24:00,180 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/092292199ce54c049fda6f28141594b5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/092292199ce54c049fda6f28141594b5 2024-11-07T15:24:00,181 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/25d0b2c138964038a27a224535a1ac1e to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/25d0b2c138964038a27a224535a1ac1e 2024-11-07T15:24:00,182 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/3a5161060d644e4fa667a1e2c90c1cf5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/3a5161060d644e4fa667a1e2c90c1cf5 2024-11-07T15:24:00,183 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/409e519c5bb14418a0698a00a2aecbbf to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/C/409e519c5bb14418a0698a00a2aecbbf 2024-11-07T15:24:00,185 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/recovered.edits/437.seqid to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493/recovered.edits/437.seqid 2024-11-07T15:24:00,185 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/b93a997d5d499fc85333e4dccefd8493 2024-11-07T15:24:00,185 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T15:24:00,187 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:00,190 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T15:24:00,192 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T15:24:00,193 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:00,193 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T15:24:00,193 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730993040193"}]},"ts":"9223372036854775807"} 2024-11-07T15:24:00,194 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T15:24:00,194 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b93a997d5d499fc85333e4dccefd8493, NAME => 'TestAcidGuarantees,,1730993013081.b93a997d5d499fc85333e4dccefd8493.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T15:24:00,194 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T15:24:00,195 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730993040194"}]},"ts":"9223372036854775807"} 2024-11-07T15:24:00,196 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T15:24:00,197 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:00,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 35 msec 2024-11-07T15:24:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-07T15:24:00,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T15:24:00,272 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:00,279 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1165652502_22 at /127.0.0.1:56860 [Waiting for operation #445] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1266856539_22 at /127.0.0.1:33226 [Waiting for operation #150] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=481 (was 465) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1283 (was 1428) 2024-11-07T15:24:00,286 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=481, ProcessCount=11, AvailableMemoryMB=1283 2024-11-07T15:24:00,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:24:00,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:24:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:00,289 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T15:24:00,289 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:00,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-07T15:24:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-07T15:24:00,290 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T15:24:00,293 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T15:24:00,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742166_1342 (size=959) 2024-11-07T15:24:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-07T15:24:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-07T15:24:00,696 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938 2024-11-07T15:24:00,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742167_1343 (size=53) 2024-11-07T15:24:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-07T15:24:01,101 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:24:01,101 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a3486aab3496e5af6c3ff570f5d35d09, disabling compactions & flushes 2024-11-07T15:24:01,101 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:01,102 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:01,102 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. after waiting 0 ms 2024-11-07T15:24:01,102 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:01,102 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:01,102 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a3486aab3496e5af6c3ff570f5d35d09: Waiting for close lock at 1730993041101Disabling compacts and flushes for region at 1730993041101Disabling writes for close at 1730993041102 (+1 ms)Writing region close event to WAL at 1730993041102Closed at 1730993041102 2024-11-07T15:24:01,103 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T15:24:01,103 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730993041103"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730993041103"}]},"ts":"1730993041103"} 2024-11-07T15:24:01,104 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T15:24:01,105 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T15:24:01,105 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993041105"}]},"ts":"1730993041105"} 2024-11-07T15:24:01,107 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T15:24:01,107 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3486aab3496e5af6c3ff570f5d35d09, ASSIGN}] 2024-11-07T15:24:01,108 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3486aab3496e5af6c3ff570f5d35d09, ASSIGN 2024-11-07T15:24:01,108 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3486aab3496e5af6c3ff570f5d35d09, ASSIGN; state=OFFLINE, location=7b3e4e8e12ef,33849,1730992951906; forceNewPlan=false, retain=false 2024-11-07T15:24:01,259 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=a3486aab3496e5af6c3ff570f5d35d09, regionState=OPENING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:01,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3486aab3496e5af6c3ff570f5d35d09, ASSIGN because future has completed 2024-11-07T15:24:01,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:24:01,415 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:01,415 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:24:01,415 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:01,415 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:24:01,415 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:01,415 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:01,417 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:01,418 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3486aab3496e5af6c3ff570f5d35d09 columnFamilyName A 2024-11-07T15:24:01,418 DEBUG [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:01,418 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] regionserver.HStore(327): Store=a3486aab3496e5af6c3ff570f5d35d09/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:24:01,418 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:01,419 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3486aab3496e5af6c3ff570f5d35d09 columnFamilyName B 2024-11-07T15:24:01,419 DEBUG [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:01,420 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] regionserver.HStore(327): Store=a3486aab3496e5af6c3ff570f5d35d09/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:24:01,420 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:01,421 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3486aab3496e5af6c3ff570f5d35d09 columnFamilyName C 2024-11-07T15:24:01,421 DEBUG [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:01,421 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] regionserver.HStore(327): Store=a3486aab3496e5af6c3ff570f5d35d09/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:24:01,421 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:01,422 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:01,422 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:01,423 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:01,423 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:01,423 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:24:01,424 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:01,426 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T15:24:01,426 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened a3486aab3496e5af6c3ff570f5d35d09; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68971446, jitterRate=0.027754634618759155}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:24:01,426 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:01,427 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for a3486aab3496e5af6c3ff570f5d35d09: Running coprocessor pre-open hook at 1730993041415Writing region info on filesystem at 1730993041415Initializing all the Stores at 1730993041416 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993041416Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993041417 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993041417Cleaning up temporary data from old regions at 1730993041423 (+6 ms)Running coprocessor post-open hooks at 1730993041426 (+3 ms)Region opened successfully at 1730993041427 (+1 ms) 2024-11-07T15:24:01,427 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., pid=91, masterSystemTime=1730993041412 2024-11-07T15:24:01,429 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:01,429 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:01,430 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=a3486aab3496e5af6c3ff570f5d35d09, regionState=OPEN, openSeqNum=2, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:01,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:24:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-07T15:24:01,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-07T15:24:01,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 in 170 msec 2024-11-07T15:24:01,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-07T15:24:01,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3486aab3496e5af6c3ff570f5d35d09, ASSIGN in 326 msec 2024-11-07T15:24:01,435 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T15:24:01,435 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993041435"}]},"ts":"1730993041435"} 2024-11-07T15:24:01,437 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T15:24:01,437 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T15:24:01,439 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-07T15:24:02,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-07T15:24:02,442 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:02,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9088e30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:02,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:02,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:02,444 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a9c651d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:02,447 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed4f1f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:02,448 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:02,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:02,450 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T15:24:02,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b3e4e8e12ef,46811,1730992951165 2024-11-07T15:24:02,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@24b00a63 2024-11-07T15:24:02,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T15:24:02,451 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T15:24:02,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:24:02,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:24:02,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:02,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742168_1344 (size=995) 2024-11-07T15:24:02,863 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T15:24:02,863 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T15:24:02,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:24:02,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3486aab3496e5af6c3ff570f5d35d09, REOPEN/MOVE}] 2024-11-07T15:24:02,868 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3486aab3496e5af6c3ff570f5d35d09, REOPEN/MOVE 2024-11-07T15:24:02,868 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a3486aab3496e5af6c3ff570f5d35d09, regionState=CLOSING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:02,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3486aab3496e5af6c3ff570f5d35d09, REOPEN/MOVE because future has completed 2024-11-07T15:24:02,870 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:24:02,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:24:03,022 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,022 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:24:03,022 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing a3486aab3496e5af6c3ff570f5d35d09, disabling compactions & flushes 2024-11-07T15:24:03,022 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:03,022 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:03,022 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. after waiting 0 ms 2024-11-07T15:24:03,022 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:03,025 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T15:24:03,026 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:03,026 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for a3486aab3496e5af6c3ff570f5d35d09: Waiting for close lock at 1730993043022Running coprocessor pre-close hooks at 1730993043022Disabling compacts and flushes for region at 1730993043022Disabling writes for close at 1730993043022Writing region close event to WAL at 1730993043023 (+1 ms)Running coprocessor post-close hooks at 1730993043026 (+3 ms)Closed at 1730993043026 2024-11-07T15:24:03,026 WARN [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: a3486aab3496e5af6c3ff570f5d35d09 to self. 2024-11-07T15:24:03,028 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,029 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a3486aab3496e5af6c3ff570f5d35d09, regionState=CLOSED 2024-11-07T15:24:03,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:24:03,032 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-07T15:24:03,032 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 in 160 msec 2024-11-07T15:24:03,032 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3486aab3496e5af6c3ff570f5d35d09, REOPEN/MOVE; state=CLOSED, location=7b3e4e8e12ef,33849,1730992951906; forceNewPlan=false, retain=true 2024-11-07T15:24:03,183 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a3486aab3496e5af6c3ff570f5d35d09, regionState=OPENING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3486aab3496e5af6c3ff570f5d35d09, REOPEN/MOVE because future has completed 2024-11-07T15:24:03,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:24:03,338 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:03,338 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:24:03,339 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,339 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:24:03,339 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,339 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,340 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,340 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3486aab3496e5af6c3ff570f5d35d09 columnFamilyName A 2024-11-07T15:24:03,341 DEBUG [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:03,342 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] regionserver.HStore(327): Store=a3486aab3496e5af6c3ff570f5d35d09/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:24:03,342 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,343 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3486aab3496e5af6c3ff570f5d35d09 columnFamilyName B 2024-11-07T15:24:03,343 DEBUG [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:03,343 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] regionserver.HStore(327): Store=a3486aab3496e5af6c3ff570f5d35d09/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:24:03,343 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,344 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3486aab3496e5af6c3ff570f5d35d09 columnFamilyName C 2024-11-07T15:24:03,344 DEBUG [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:03,344 INFO [StoreOpener-a3486aab3496e5af6c3ff570f5d35d09-1 {}] regionserver.HStore(327): Store=a3486aab3496e5af6c3ff570f5d35d09/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:24:03,344 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,345 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,346 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,347 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,347 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,347 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:24:03,348 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,349 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened a3486aab3496e5af6c3ff570f5d35d09; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73606288, jitterRate=0.09681916236877441}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:24:03,349 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,349 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for a3486aab3496e5af6c3ff570f5d35d09: Running coprocessor pre-open hook at 1730993043339Writing region info on filesystem at 1730993043339Initializing all the Stores at 1730993043339Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993043339Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993043340 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993043340Cleaning up temporary data from old regions at 1730993043347 (+7 ms)Running coprocessor post-open hooks at 1730993043349 (+2 ms)Region opened successfully at 1730993043349 2024-11-07T15:24:03,350 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., pid=96, masterSystemTime=1730993043336 2024-11-07T15:24:03,352 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:03,352 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:03,352 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a3486aab3496e5af6c3ff570f5d35d09, regionState=OPEN, openSeqNum=5, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:24:03,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-07T15:24:03,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 in 170 msec 2024-11-07T15:24:03,357 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-07T15:24:03,357 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3486aab3496e5af6c3ff570f5d35d09, REOPEN/MOVE in 488 msec 2024-11-07T15:24:03,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-07T15:24:03,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-07T15:24:03,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 906 msec 2024-11-07T15:24:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-07T15:24:03,361 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:03,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12688331, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,362 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:03,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:03,363 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cf804e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:03,363 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:03,364 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:03,364 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b41c6a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1456e4e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,365 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:03,365 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:03,365 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac33453, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:03,367 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63804b4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32da4b0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,368 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:03,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:03,368 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@229fc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:03,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:03,369 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:03,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577e4c2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e8e9c8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,370 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:03,370 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:03,371 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:03,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:03,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:03,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683f94da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:03,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:03,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:03,372 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:03,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74aedbc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a91e16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,373 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:03,373 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:03,374 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:03,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:03,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:03,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b3f1521, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:03,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:03,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:03,375 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c7c4436, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bdde2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,376 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:03,376 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:03,377 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14bdf70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:03,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:03,378 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:03,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c46f17a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7608485b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,379 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:03,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:03,380 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:03,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:03,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:03,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e58e80e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:03,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:03,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:03,381 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:03,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39fe68ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf0e288, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,382 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:03,382 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:03,383 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:03,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:03,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:03,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eaf2cd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:03,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:03,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:03,385 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ff204f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3f81f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,386 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:03,386 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:03,387 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:03,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:03,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:03,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26484d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:03,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:03,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:03,388 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:03,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577abf21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19dc265d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,389 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:03,389 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:03,390 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:03,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:03,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:03,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78eb2bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:03,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:03,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:03,392 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4167cdc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:03,393 DEBUG [Thread-1565 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:03,393 DEBUG [Thread-1569 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:03,393 DEBUG [Thread-1571 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:03,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:03,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:03,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:03,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:03,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:03,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:03,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:03,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:03,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:24:03,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:24:03,397 DEBUG [Thread-1576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:03,397 DEBUG [Thread-1574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:03,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:24:03,398 DEBUG [Thread-1574 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:03,398 DEBUG [Thread-1574 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:03,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:03,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:03,400 DEBUG [Thread-1567 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:03,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:03,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:24:03,403 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:24:03,404 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:03,404 DEBUG [Thread-1582 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-07T15:24:03,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:03,405 DEBUG [Thread-1578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:03,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T15:24:03,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:03,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:03,407 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:03,407 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:03,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:24:03,408 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:03,408 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:03,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:24:03,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:03,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:24:03,411 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:03,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3486aab3496e5af6c3ff570f5d35d09 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T15:24:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,418 DEBUG [Thread-1580 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:03,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:03,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:03,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:03,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:24:03,436 DEBUG [Thread-1563 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:03,439 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:24:03,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074690ba2daecc4bc78096ceedbf594b84_a3486aab3496e5af6c3ff570f5d35d09 is 50, key is test_row_0/A:col10/1730993043408/Put/seqid=0 2024-11-07T15:24:03,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742169_1345 (size=14594) 2024-11-07T15:24:03,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993103463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993103466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:03,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993103467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:03,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46108 deadline: 1730993103468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:03,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46110 deadline: 1730993103470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T15:24:03,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-07T15:24:03,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:03,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:03,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:03,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:03,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:03,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993103582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:03,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993103582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:03,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993103582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:03,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46108 deadline: 1730993103591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46110 deadline: 1730993103592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:03,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-07T15:24:03,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:03,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:03,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:03,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:03,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T15:24:03,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993103790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993103790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:03,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993103801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:03,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46108 deadline: 1730993103811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46110 deadline: 1730993103811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:03,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-07T15:24:03,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:03,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:03,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:03,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:03,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:03,876 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:03,879 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074690ba2daecc4bc78096ceedbf594b84_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074690ba2daecc4bc78096ceedbf594b84_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:03,880 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/4aba4a6a91a94446a5bfd17bb94e0979, store: [table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:03,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/4aba4a6a91a94446a5bfd17bb94e0979 is 175, key is test_row_0/A:col10/1730993043408/Put/seqid=0 2024-11-07T15:24:03,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742170_1346 (size=39549) 2024-11-07T15:24:03,886 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/4aba4a6a91a94446a5bfd17bb94e0979 2024-11-07T15:24:03,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/88a8d58857f94bc9ae33e0b787280afd is 50, key is test_row_0/B:col10/1730993043408/Put/seqid=0 2024-11-07T15:24:03,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742171_1347 (size=12001) 2024-11-07T15:24:03,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/88a8d58857f94bc9ae33e0b787280afd 2024-11-07T15:24:03,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/0d1a724e1c164b85b4de0857ce8b9e67 is 50, key is test_row_0/C:col10/1730993043408/Put/seqid=0 2024-11-07T15:24:03,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742172_1348 (size=12001) 2024-11-07T15:24:03,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/0d1a724e1c164b85b4de0857ce8b9e67 2024-11-07T15:24:03,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/4aba4a6a91a94446a5bfd17bb94e0979 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/4aba4a6a91a94446a5bfd17bb94e0979 2024-11-07T15:24:03,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/4aba4a6a91a94446a5bfd17bb94e0979, entries=200, sequenceid=17, filesize=38.6 K 2024-11-07T15:24:03,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/88a8d58857f94bc9ae33e0b787280afd as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/88a8d58857f94bc9ae33e0b787280afd 2024-11-07T15:24:03,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/88a8d58857f94bc9ae33e0b787280afd, entries=150, sequenceid=17, filesize=11.7 K 2024-11-07T15:24:03,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/0d1a724e1c164b85b4de0857ce8b9e67 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/0d1a724e1c164b85b4de0857ce8b9e67 2024-11-07T15:24:03,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/0d1a724e1c164b85b4de0857ce8b9e67, entries=150, sequenceid=17, filesize=11.7 K 2024-11-07T15:24:03,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a3486aab3496e5af6c3ff570f5d35d09 in 549ms, sequenceid=17, compaction requested=false 2024-11-07T15:24:03,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:04,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-07T15:24:04,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:04,021 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing a3486aab3496e5af6c3ff570f5d35d09 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T15:24:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076e7dd8bb668b41a2843538f5fdb1ff93_a3486aab3496e5af6c3ff570f5d35d09 is 50, key is test_row_0/A:col10/1730993043451/Put/seqid=0 2024-11-07T15:24:04,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T15:24:04,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742173_1349 (size=12154) 2024-11-07T15:24:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:04,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:04,115 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T15:24:04,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993104117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:04,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46110 deadline: 1730993104120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:04,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:04,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46108 deadline: 1730993104120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:04,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:04,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993104123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:04,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:04,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993104123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:04,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993104230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:04,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993104241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993104241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:04,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:04,442 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076e7dd8bb668b41a2843538f5fdb1ff93_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076e7dd8bb668b41a2843538f5fdb1ff93_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:04,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993104440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:04,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:04,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/6e9b8057026f4bbcb2b85fb787e27d8b, store: [table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:04,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/6e9b8057026f4bbcb2b85fb787e27d8b is 175, key is test_row_0/A:col10/1730993043451/Put/seqid=0 2024-11-07T15:24:04,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993104451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:04,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993104452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:04,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:04,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,459 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:04,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742174_1350 (size=30955) 2024-11-07T15:24:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T15:24:04,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46110 deadline: 1730993104630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:04,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46108 deadline: 1730993104640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:04,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993104750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:04,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:04,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993104761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:04,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:04,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993104771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:04,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:04,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:04,878 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/6e9b8057026f4bbcb2b85fb787e27d8b 2024-11-07T15:24:04,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/c1d5d7d9a2e644f0928c2d681391bade is 50, key is test_row_0/B:col10/1730993043451/Put/seqid=0 2024-11-07T15:24:04,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742175_1351 (size=12001) 2024-11-07T15:24:05,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993105261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:05,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993105271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:05,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993105281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:05,294 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/c1d5d7d9a2e644f0928c2d681391bade 2024-11-07T15:24:05,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/7b848316cc0a476ea80de55b78f9c360 is 50, key is test_row_0/C:col10/1730993043451/Put/seqid=0 2024-11-07T15:24:05,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742176_1352 (size=12001) 2024-11-07T15:24:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T15:24:05,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46110 deadline: 1730993105651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:05,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46108 deadline: 1730993105651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:05,658 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in a3486aab3496e5af6c3ff570f5d35d09 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:05,658 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in a3486aab3496e5af6c3ff570f5d35d09 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:05,709 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/7b848316cc0a476ea80de55b78f9c360 2024-11-07T15:24:05,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/6e9b8057026f4bbcb2b85fb787e27d8b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6e9b8057026f4bbcb2b85fb787e27d8b 2024-11-07T15:24:05,715 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6e9b8057026f4bbcb2b85fb787e27d8b, entries=150, sequenceid=40, filesize=30.2 K 2024-11-07T15:24:05,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/c1d5d7d9a2e644f0928c2d681391bade as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/c1d5d7d9a2e644f0928c2d681391bade 2024-11-07T15:24:05,720 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/c1d5d7d9a2e644f0928c2d681391bade, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T15:24:05,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/7b848316cc0a476ea80de55b78f9c360 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/7b848316cc0a476ea80de55b78f9c360 2024-11-07T15:24:05,725 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/7b848316cc0a476ea80de55b78f9c360, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T15:24:05,727 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for a3486aab3496e5af6c3ff570f5d35d09 in 1705ms, sequenceid=40, compaction requested=false 2024-11-07T15:24:05,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:05,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:05,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-07T15:24:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-07T15:24:05,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-07T15:24:05,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3200 sec 2024-11-07T15:24:05,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.3270 sec 2024-11-07T15:24:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:06,276 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:06,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3486aab3496e5af6c3ff570f5d35d09 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T15:24:06,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f060b40c71de477586dddac3954c26c1_a3486aab3496e5af6c3ff570f5d35d09 is 50, key is test_row_0/A:col10/1730993044112/Put/seqid=0 2024-11-07T15:24:06,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742177_1353 (size=14594) 2024-11-07T15:24:06,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993106339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:06,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993106340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:06,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993106342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:06,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993106451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:06,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:06,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993106460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993106460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:06,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993106661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:06,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993106671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:06,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993106680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:06,715 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:06,722 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f060b40c71de477586dddac3954c26c1_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f060b40c71de477586dddac3954c26c1_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:06,725 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/7e2ecf043aa746d78a507890a4308a16, store: [table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:06,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/7e2ecf043aa746d78a507890a4308a16 is 175, key is test_row_0/A:col10/1730993044112/Put/seqid=0 2024-11-07T15:24:06,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742178_1354 (size=39549) 2024-11-07T15:24:06,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993106971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:06,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:06,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993106980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:06,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993106990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:06,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:07,128 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/7e2ecf043aa746d78a507890a4308a16 2024-11-07T15:24:07,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/7f36b3fa05b749febdb934ad962a1308 is 50, key is test_row_0/B:col10/1730993044112/Put/seqid=0 2024-11-07T15:24:07,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742179_1355 (size=12001) 2024-11-07T15:24:07,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993107490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:07,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993107500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:07,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:07,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:07,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:07,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993107501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:07,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:07,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:07,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:07,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/7f36b3fa05b749febdb934ad962a1308 2024-11-07T15:24:07,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/fce4da5ced6e4604974b3f71df3ed2a4 is 50, key is test_row_0/C:col10/1730993044112/Put/seqid=0 2024-11-07T15:24:07,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742180_1356 (size=12001) 2024-11-07T15:24:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-07T15:24:07,560 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:07,561 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-07T15:24:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T15:24:07,562 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:07,563 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:07,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:07,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T15:24:07,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46110 deadline: 1730993107671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:07,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46108 deadline: 1730993107671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:07,678 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in a3486aab3496e5af6c3ff570f5d35d09 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:07,679 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in a3486aab3496e5af6c3ff570f5d35d09 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:07,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:07,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T15:24:07,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:07,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:07,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:07,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:07,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:07,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:07,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T15:24:07,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:07,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:07,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:07,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:07,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:07,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:07,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T15:24:07,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/fce4da5ced6e4604974b3f71df3ed2a4 2024-11-07T15:24:07,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/7e2ecf043aa746d78a507890a4308a16 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/7e2ecf043aa746d78a507890a4308a16 2024-11-07T15:24:07,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/7e2ecf043aa746d78a507890a4308a16, entries=200, sequenceid=54, filesize=38.6 K 2024-11-07T15:24:07,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/7f36b3fa05b749febdb934ad962a1308 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/7f36b3fa05b749febdb934ad962a1308 2024-11-07T15:24:07,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/7f36b3fa05b749febdb934ad962a1308, entries=150, sequenceid=54, filesize=11.7 K 2024-11-07T15:24:07,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/fce4da5ced6e4604974b3f71df3ed2a4 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/fce4da5ced6e4604974b3f71df3ed2a4 2024-11-07T15:24:07,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/fce4da5ced6e4604974b3f71df3ed2a4, entries=150, sequenceid=54, filesize=11.7 K 2024-11-07T15:24:07,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a3486aab3496e5af6c3ff570f5d35d09 in 1692ms, sequenceid=54, compaction requested=true 2024-11-07T15:24:07,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:07,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:24:07,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:07,968 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:07,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:24:07,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:07,968 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:07,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:24:07,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:07,969 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:07,969 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:07,969 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/A is initiating minor compaction (all files) 2024-11-07T15:24:07,969 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/B is initiating minor compaction (all files) 2024-11-07T15:24:07,969 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/B in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:07,969 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/A in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:07,969 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/88a8d58857f94bc9ae33e0b787280afd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/c1d5d7d9a2e644f0928c2d681391bade, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/7f36b3fa05b749febdb934ad962a1308] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=35.2 K 2024-11-07T15:24:07,969 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/4aba4a6a91a94446a5bfd17bb94e0979, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6e9b8057026f4bbcb2b85fb787e27d8b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/7e2ecf043aa746d78a507890a4308a16] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=107.5 K 2024-11-07T15:24:07,969 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:07,969 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/4aba4a6a91a94446a5bfd17bb94e0979, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6e9b8057026f4bbcb2b85fb787e27d8b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/7e2ecf043aa746d78a507890a4308a16] 2024-11-07T15:24:07,970 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 88a8d58857f94bc9ae33e0b787280afd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1730993043408 2024-11-07T15:24:07,970 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4aba4a6a91a94446a5bfd17bb94e0979, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1730993043400 2024-11-07T15:24:07,970 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting c1d5d7d9a2e644f0928c2d681391bade, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730993043451 2024-11-07T15:24:07,970 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e9b8057026f4bbcb2b85fb787e27d8b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730993043451 2024-11-07T15:24:07,970 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f36b3fa05b749febdb934ad962a1308, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730993044112 2024-11-07T15:24:07,971 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e2ecf043aa746d78a507890a4308a16, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730993044112 2024-11-07T15:24:07,976 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:07,979 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#B#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:07,980 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/0b9b2f84d40d40e0921b02988fcf066b is 50, key is test_row_0/B:col10/1730993044112/Put/seqid=0 2024-11-07T15:24:07,985 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107be781f477fa442758b8612826c01d779_a3486aab3496e5af6c3ff570f5d35d09 store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:07,986 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107be781f477fa442758b8612826c01d779_a3486aab3496e5af6c3ff570f5d35d09, store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:07,986 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107be781f477fa442758b8612826c01d779_a3486aab3496e5af6c3ff570f5d35d09 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:07,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742181_1357 (size=12104) 2024-11-07T15:24:07,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742182_1358 (size=4469) 2024-11-07T15:24:08,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T15:24:08,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:08,020 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing a3486aab3496e5af6c3ff570f5d35d09 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T15:24:08,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110780646c93ee894de996a687a5bdbd168b_a3486aab3496e5af6c3ff570f5d35d09 is 50, key is test_row_0/A:col10/1730993046341/Put/seqid=0 2024-11-07T15:24:08,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742183_1359 (size=12154) 2024-11-07T15:24:08,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:08,037 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110780646c93ee894de996a687a5bdbd168b_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110780646c93ee894de996a687a5bdbd168b_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:08,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/a6865629762f4206b491d728382f992d, store: [table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:08,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/a6865629762f4206b491d728382f992d is 175, key is test_row_0/A:col10/1730993046341/Put/seqid=0 2024-11-07T15:24:08,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742184_1360 (size=30955) 2024-11-07T15:24:08,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T15:24:08,395 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#A#compaction#301 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:08,395 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/0b9b2f84d40d40e0921b02988fcf066b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/0b9b2f84d40d40e0921b02988fcf066b 2024-11-07T15:24:08,395 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/3038ff7e8b7c40fc9ad19c4be02c66bf is 175, key is test_row_0/A:col10/1730993044112/Put/seqid=0 2024-11-07T15:24:08,402 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/B of a3486aab3496e5af6c3ff570f5d35d09 into 0b9b2f84d40d40e0921b02988fcf066b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:08,402 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:08,402 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/B, priority=13, startTime=1730993047968; duration=0sec 2024-11-07T15:24:08,402 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:08,402 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:B 2024-11-07T15:24:08,402 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:08,403 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:08,403 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/C is initiating minor compaction (all files) 2024-11-07T15:24:08,403 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/C in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:08,403 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/0d1a724e1c164b85b4de0857ce8b9e67, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/7b848316cc0a476ea80de55b78f9c360, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/fce4da5ced6e4604974b3f71df3ed2a4] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=35.2 K 2024-11-07T15:24:08,404 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d1a724e1c164b85b4de0857ce8b9e67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1730993043408 2024-11-07T15:24:08,404 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b848316cc0a476ea80de55b78f9c360, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730993043451 2024-11-07T15:24:08,405 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting fce4da5ced6e4604974b3f71df3ed2a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730993044112 2024-11-07T15:24:08,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742185_1361 (size=31058) 2024-11-07T15:24:08,412 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/3038ff7e8b7c40fc9ad19c4be02c66bf as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/3038ff7e8b7c40fc9ad19c4be02c66bf 2024-11-07T15:24:08,416 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/A of a3486aab3496e5af6c3ff570f5d35d09 into 3038ff7e8b7c40fc9ad19c4be02c66bf(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:08,416 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:08,416 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/A, priority=13, startTime=1730993047968; duration=0sec 2024-11-07T15:24:08,416 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:08,416 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:A 2024-11-07T15:24:08,426 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#C#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:08,427 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/07b044bc4681437d8ff4931d9bcb5309 is 50, key is test_row_0/C:col10/1730993044112/Put/seqid=0 2024-11-07T15:24:08,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742186_1362 (size=12104) 2024-11-07T15:24:08,446 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/a6865629762f4206b491d728382f992d 2024-11-07T15:24:08,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/77bf5548d09a40b2b7204beb60008922 is 50, key is test_row_0/B:col10/1730993046341/Put/seqid=0 2024-11-07T15:24:08,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742187_1363 (size=12001) 2024-11-07T15:24:08,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:08,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993108537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:08,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993108538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:08,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993108539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:08,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993108651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:08,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993108651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:08,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993108651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:08,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T15:24:08,836 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/07b044bc4681437d8ff4931d9bcb5309 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/07b044bc4681437d8ff4931d9bcb5309 2024-11-07T15:24:08,841 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/C of a3486aab3496e5af6c3ff570f5d35d09 into 07b044bc4681437d8ff4931d9bcb5309(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:08,841 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:08,842 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/C, priority=13, startTime=1730993047968; duration=0sec 2024-11-07T15:24:08,842 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:08,842 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:C 2024-11-07T15:24:08,862 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/77bf5548d09a40b2b7204beb60008922 2024-11-07T15:24:08,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993108861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993108861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993108861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:08,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/38847533e61143f689e8b4c15984d445 is 50, key is test_row_0/C:col10/1730993046341/Put/seqid=0 2024-11-07T15:24:08,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742188_1364 (size=12001) 2024-11-07T15:24:09,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993109171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993109171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993109171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:09,282 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/38847533e61143f689e8b4c15984d445 2024-11-07T15:24:09,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/a6865629762f4206b491d728382f992d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/a6865629762f4206b491d728382f992d 2024-11-07T15:24:09,290 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/a6865629762f4206b491d728382f992d, entries=150, sequenceid=76, filesize=30.2 K 2024-11-07T15:24:09,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/77bf5548d09a40b2b7204beb60008922 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/77bf5548d09a40b2b7204beb60008922 2024-11-07T15:24:09,294 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/77bf5548d09a40b2b7204beb60008922, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T15:24:09,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/38847533e61143f689e8b4c15984d445 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/38847533e61143f689e8b4c15984d445 2024-11-07T15:24:09,299 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/38847533e61143f689e8b4c15984d445, entries=150, sequenceid=76, filesize=11.7 K 2024-11-07T15:24:09,300 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for a3486aab3496e5af6c3ff570f5d35d09 in 1280ms, sequenceid=76, compaction requested=false 2024-11-07T15:24:09,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:09,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:09,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-07T15:24:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-07T15:24:09,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-07T15:24:09,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7390 sec 2024-11-07T15:24:09,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.7430 sec 2024-11-07T15:24:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:09,685 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:09,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3486aab3496e5af6c3ff570f5d35d09 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T15:24:09,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070da166fa06c649ff95a5bb62484d6437_a3486aab3496e5af6c3ff570f5d35d09 is 50, key is test_row_0/A:col10/1730993048527/Put/seqid=0 2024-11-07T15:24:09,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742189_1365 (size=14594) 2024-11-07T15:24:09,702 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:09,705 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070da166fa06c649ff95a5bb62484d6437_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070da166fa06c649ff95a5bb62484d6437_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:09,706 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/aff2137755a74eafb86675c6682dae97, store: [table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:09,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/aff2137755a74eafb86675c6682dae97 is 175, key is test_row_0/A:col10/1730993048527/Put/seqid=0 2024-11-07T15:24:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T15:24:09,711 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:09,711 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-07T15:24:09,713 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T15:24:09,713 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:09,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:09,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742190_1366 (size=39549) 2024-11-07T15:24:09,718 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/aff2137755a74eafb86675c6682dae97 2024-11-07T15:24:09,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/0ca097b54c184bc6b5d517ddeaf19e45 is 50, key is test_row_0/B:col10/1730993048527/Put/seqid=0 2024-11-07T15:24:09,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742191_1367 (size=12001) 2024-11-07T15:24:09,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:09,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993109753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993109752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:09,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993109754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:09,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T15:24:09,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T15:24:09,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:09,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:09,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:09,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:09,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:09,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993109871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993109871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:09,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993109871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:10,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T15:24:10,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:10,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:10,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:10,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:10,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T15:24:10,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993110080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:10,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993110080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993110081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:10,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/0ca097b54c184bc6b5d517ddeaf19e45 2024-11-07T15:24:10,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/4609a090939d4d7180fe7496e9b7bd88 is 50, key is test_row_0/C:col10/1730993048527/Put/seqid=0 2024-11-07T15:24:10,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742192_1368 (size=12001) 2024-11-07T15:24:10,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T15:24:10,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:10,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:10,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:10,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:10,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:10,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T15:24:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:10,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T15:24:10,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993110391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993110391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:10,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993110401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:10,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T15:24:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:10,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:10,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:10,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/4609a090939d4d7180fe7496e9b7bd88 2024-11-07T15:24:10,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/aff2137755a74eafb86675c6682dae97 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/aff2137755a74eafb86675c6682dae97 2024-11-07T15:24:10,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/aff2137755a74eafb86675c6682dae97, entries=200, sequenceid=94, filesize=38.6 K 2024-11-07T15:24:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/0ca097b54c184bc6b5d517ddeaf19e45 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/0ca097b54c184bc6b5d517ddeaf19e45 2024-11-07T15:24:10,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/0ca097b54c184bc6b5d517ddeaf19e45, entries=150, sequenceid=94, filesize=11.7 K 2024-11-07T15:24:10,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/4609a090939d4d7180fe7496e9b7bd88 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/4609a090939d4d7180fe7496e9b7bd88 2024-11-07T15:24:10,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/4609a090939d4d7180fe7496e9b7bd88, entries=150, sequenceid=94, filesize=11.7 K 2024-11-07T15:24:10,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for a3486aab3496e5af6c3ff570f5d35d09 in 886ms, sequenceid=94, compaction requested=true 2024-11-07T15:24:10,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:10,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:24:10,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:10,572 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:10,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:24:10,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:10,572 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:10,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:24:10,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:10,573 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:10,573 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:10,573 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/B is initiating minor compaction (all files) 2024-11-07T15:24:10,573 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/A is initiating minor compaction (all files) 2024-11-07T15:24:10,573 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/B in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:10,573 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/A in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:10,573 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/0b9b2f84d40d40e0921b02988fcf066b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/77bf5548d09a40b2b7204beb60008922, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/0ca097b54c184bc6b5d517ddeaf19e45] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=35.3 K 2024-11-07T15:24:10,573 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/3038ff7e8b7c40fc9ad19c4be02c66bf, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/a6865629762f4206b491d728382f992d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/aff2137755a74eafb86675c6682dae97] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=99.2 K 2024-11-07T15:24:10,573 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:10,573 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/3038ff7e8b7c40fc9ad19c4be02c66bf, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/a6865629762f4206b491d728382f992d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/aff2137755a74eafb86675c6682dae97] 2024-11-07T15:24:10,575 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3038ff7e8b7c40fc9ad19c4be02c66bf, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730993044112 2024-11-07T15:24:10,575 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b9b2f84d40d40e0921b02988fcf066b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730993044112 2024-11-07T15:24:10,575 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 77bf5548d09a40b2b7204beb60008922, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730993046329 2024-11-07T15:24:10,575 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6865629762f4206b491d728382f992d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730993046329 2024-11-07T15:24:10,575 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ca097b54c184bc6b5d517ddeaf19e45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730993048527 2024-11-07T15:24:10,575 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting aff2137755a74eafb86675c6682dae97, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730993048527 2024-11-07T15:24:10,583 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#B#compaction#310 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:10,583 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/00fb693f37ec4cdeb0831abe1f525aaa is 50, key is test_row_0/B:col10/1730993048527/Put/seqid=0 2024-11-07T15:24:10,585 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:10,596 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107fbfa2583e51c4e6683a63a245c87e30f_a3486aab3496e5af6c3ff570f5d35d09 store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:10,599 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107fbfa2583e51c4e6683a63a245c87e30f_a3486aab3496e5af6c3ff570f5d35d09, store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:10,599 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107fbfa2583e51c4e6683a63a245c87e30f_a3486aab3496e5af6c3ff570f5d35d09 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:10,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742193_1369 (size=12207) 2024-11-07T15:24:10,606 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/00fb693f37ec4cdeb0831abe1f525aaa as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/00fb693f37ec4cdeb0831abe1f525aaa 2024-11-07T15:24:10,614 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/B of a3486aab3496e5af6c3ff570f5d35d09 into 00fb693f37ec4cdeb0831abe1f525aaa(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:10,614 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:10,614 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/B, priority=13, startTime=1730993050572; duration=0sec 2024-11-07T15:24:10,614 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:10,614 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:B 2024-11-07T15:24:10,614 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:10,616 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:10,616 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/C is initiating minor compaction (all files) 2024-11-07T15:24:10,616 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/C in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:10,616 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/07b044bc4681437d8ff4931d9bcb5309, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/38847533e61143f689e8b4c15984d445, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/4609a090939d4d7180fe7496e9b7bd88] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=35.3 K 2024-11-07T15:24:10,617 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 07b044bc4681437d8ff4931d9bcb5309, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730993044112 2024-11-07T15:24:10,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742194_1370 (size=4469) 2024-11-07T15:24:10,617 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 38847533e61143f689e8b4c15984d445, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1730993046329 2024-11-07T15:24:10,618 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 4609a090939d4d7180fe7496e9b7bd88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730993048527 2024-11-07T15:24:10,626 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#C#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:10,627 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/6c7aeab6003f4c08bc94d0fe7c06f5b2 is 50, key is test_row_0/C:col10/1730993048527/Put/seqid=0 2024-11-07T15:24:10,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T15:24:10,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:10,630 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing a3486aab3496e5af6c3ff570f5d35d09 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T15:24:10,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110785283bf9a02d45e98bab4cffcdc86ed2_a3486aab3496e5af6c3ff570f5d35d09 is 50, key is test_row_0/A:col10/1730993049750/Put/seqid=0 2024-11-07T15:24:10,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742195_1371 (size=12207) 2024-11-07T15:24:10,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742196_1372 (size=12154) 2024-11-07T15:24:10,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:10,669 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110785283bf9a02d45e98bab4cffcdc86ed2_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110785283bf9a02d45e98bab4cffcdc86ed2_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:10,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/540c60ac58b34317a72dbc187e54350f, store: [table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:10,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/540c60ac58b34317a72dbc187e54350f is 175, key is test_row_0/A:col10/1730993049750/Put/seqid=0 2024-11-07T15:24:10,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742197_1373 (size=30955) 2024-11-07T15:24:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T15:24:10,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:10,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993110950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:10,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993110951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:10,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993110952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:10,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:10,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,018 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#A#compaction#311 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:11,019 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/6d41a8f509ad47da8d22deb59d50119a is 175, key is test_row_0/A:col10/1730993048527/Put/seqid=0 2024-11-07T15:24:11,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742198_1374 (size=31161) 2024-11-07T15:24:11,053 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/6c7aeab6003f4c08bc94d0fe7c06f5b2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/6c7aeab6003f4c08bc94d0fe7c06f5b2 2024-11-07T15:24:11,056 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/C of a3486aab3496e5af6c3ff570f5d35d09 into 6c7aeab6003f4c08bc94d0fe7c06f5b2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:11,057 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:11,057 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/C, priority=13, startTime=1730993050572; duration=0sec 2024-11-07T15:24:11,057 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:11,057 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:C 2024-11-07T15:24:11,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993111070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993111070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993111070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,082 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/540c60ac58b34317a72dbc187e54350f 2024-11-07T15:24:11,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/63c7418b30264a008ef4a3ba6c5d3d3d is 50, key is test_row_0/B:col10/1730993049750/Put/seqid=0 2024-11-07T15:24:11,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742199_1375 (size=12001) 2024-11-07T15:24:11,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993111281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993111281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993111281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,428 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/6d41a8f509ad47da8d22deb59d50119a as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6d41a8f509ad47da8d22deb59d50119a 2024-11-07T15:24:11,431 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/A of a3486aab3496e5af6c3ff570f5d35d09 into 6d41a8f509ad47da8d22deb59d50119a(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:11,431 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:11,431 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/A, priority=13, startTime=1730993050571; duration=0sec 2024-11-07T15:24:11,432 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:11,432 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:A 2024-11-07T15:24:11,494 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/63c7418b30264a008ef4a3ba6c5d3d3d 2024-11-07T15:24:11,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/5d7ce89540ad4591a3d612c64dac0575 is 50, key is test_row_0/C:col10/1730993049750/Put/seqid=0 2024-11-07T15:24:11,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742200_1376 (size=12001) 2024-11-07T15:24:11,516 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/5d7ce89540ad4591a3d612c64dac0575 2024-11-07T15:24:11,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/540c60ac58b34317a72dbc187e54350f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/540c60ac58b34317a72dbc187e54350f 2024-11-07T15:24:11,527 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/540c60ac58b34317a72dbc187e54350f, entries=150, sequenceid=116, filesize=30.2 K 2024-11-07T15:24:11,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/63c7418b30264a008ef4a3ba6c5d3d3d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/63c7418b30264a008ef4a3ba6c5d3d3d 2024-11-07T15:24:11,532 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/63c7418b30264a008ef4a3ba6c5d3d3d, entries=150, sequenceid=116, filesize=11.7 K 2024-11-07T15:24:11,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/5d7ce89540ad4591a3d612c64dac0575 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/5d7ce89540ad4591a3d612c64dac0575 2024-11-07T15:24:11,536 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/5d7ce89540ad4591a3d612c64dac0575, entries=150, sequenceid=116, filesize=11.7 K 2024-11-07T15:24:11,537 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for a3486aab3496e5af6c3ff570f5d35d09 in 907ms, sequenceid=116, compaction requested=false 2024-11-07T15:24:11,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:11,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:11,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-07T15:24:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-07T15:24:11,540 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-07T15:24:11,540 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8250 sec 2024-11-07T15:24:11,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.8290 sec 2024-11-07T15:24:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:11,598 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:11,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3486aab3496e5af6c3ff570f5d35d09 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T15:24:11,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070bc312dc6ab246fba69837f460e7a06e_a3486aab3496e5af6c3ff570f5d35d09 is 50, key is test_row_0/A:col10/1730993050951/Put/seqid=0 2024-11-07T15:24:11,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742201_1377 (size=17184) 2024-11-07T15:24:11,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993111641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993111644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993111645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46110 deadline: 1730993111700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,709 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in a3486aab3496e5af6c3ff570f5d35d09 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46108 deadline: 1730993111720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,728 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in a3486aab3496e5af6c3ff570f5d35d09 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:11,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993111751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993111761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993111761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T15:24:11,861 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:11,862 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-07T15:24:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T15:24:11,863 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:11,864 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:11,864 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:11,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993111961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T15:24:11,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993111980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:11,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993111980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:12,014 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:12,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:24:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,019 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070bc312dc6ab246fba69837f460e7a06e_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070bc312dc6ab246fba69837f460e7a06e_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:12,020 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/2e8b7a2a17a64c5998fba2510786dd77, store: [table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:12,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/2e8b7a2a17a64c5998fba2510786dd77 is 175, key is test_row_0/A:col10/1730993050951/Put/seqid=0 2024-11-07T15:24:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742202_1378 (size=48289) 2024-11-07T15:24:12,030 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/2e8b7a2a17a64c5998fba2510786dd77 2024-11-07T15:24:12,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/1a508dee6ae94f6680b6876a01674446 is 50, key is test_row_0/B:col10/1730993050951/Put/seqid=0 2024-11-07T15:24:12,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742203_1379 (size=12101) 2024-11-07T15:24:12,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:24:12,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:12,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T15:24:12,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993112281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:12,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993112301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:12,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993112301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:12,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:24:12,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:12,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:24:12,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:12,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T15:24:12,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/1a508dee6ae94f6680b6876a01674446 2024-11-07T15:24:12,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/433dfe8c88ed46f5acf219d8851a3ce4 is 50, key is test_row_0/C:col10/1730993050951/Put/seqid=0 2024-11-07T15:24:12,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742204_1380 (size=12101) 2024-11-07T15:24:12,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:24:12,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:12,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:12,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993112801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:12,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:24:12,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:12,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:12,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:12,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993112810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:12,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993112820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:12,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:12,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:12,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:12,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/433dfe8c88ed46f5acf219d8851a3ce4 2024-11-07T15:24:12,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/2e8b7a2a17a64c5998fba2510786dd77 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/2e8b7a2a17a64c5998fba2510786dd77 2024-11-07T15:24:12,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/2e8b7a2a17a64c5998fba2510786dd77, entries=250, sequenceid=135, filesize=47.2 K 2024-11-07T15:24:12,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/1a508dee6ae94f6680b6876a01674446 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/1a508dee6ae94f6680b6876a01674446 2024-11-07T15:24:12,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/1a508dee6ae94f6680b6876a01674446, entries=150, sequenceid=135, filesize=11.8 K 2024-11-07T15:24:12,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/433dfe8c88ed46f5acf219d8851a3ce4 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/433dfe8c88ed46f5acf219d8851a3ce4 2024-11-07T15:24:12,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/433dfe8c88ed46f5acf219d8851a3ce4, entries=150, sequenceid=135, filesize=11.8 K 2024-11-07T15:24:12,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a3486aab3496e5af6c3ff570f5d35d09 in 1336ms, sequenceid=135, compaction requested=true 2024-11-07T15:24:12,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:12,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:24:12,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:12,934 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:12,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:24:12,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:12,934 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:12,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:24:12,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:12,935 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:12,935 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/B is initiating minor compaction (all files) 2024-11-07T15:24:12,935 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/B in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,935 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/00fb693f37ec4cdeb0831abe1f525aaa, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/63c7418b30264a008ef4a3ba6c5d3d3d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/1a508dee6ae94f6680b6876a01674446] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=35.5 K 2024-11-07T15:24:12,936 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:12,936 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/A is initiating minor compaction (all files) 2024-11-07T15:24:12,936 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 00fb693f37ec4cdeb0831abe1f525aaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730993048527 2024-11-07T15:24:12,936 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/A in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,936 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6d41a8f509ad47da8d22deb59d50119a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/540c60ac58b34317a72dbc187e54350f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/2e8b7a2a17a64c5998fba2510786dd77] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=107.8 K 2024-11-07T15:24:12,936 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,936 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6d41a8f509ad47da8d22deb59d50119a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/540c60ac58b34317a72dbc187e54350f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/2e8b7a2a17a64c5998fba2510786dd77] 2024-11-07T15:24:12,936 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 63c7418b30264a008ef4a3ba6c5d3d3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1730993049737 2024-11-07T15:24:12,936 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a508dee6ae94f6680b6876a01674446, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1730993050948 2024-11-07T15:24:12,936 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d41a8f509ad47da8d22deb59d50119a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730993048527 2024-11-07T15:24:12,937 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 540c60ac58b34317a72dbc187e54350f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1730993049737 2024-11-07T15:24:12,937 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e8b7a2a17a64c5998fba2510786dd77, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1730993050932 2024-11-07T15:24:12,944 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:12,950 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#B#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:12,951 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/e2753bcb363f48f28f97f87acf346bf7 is 50, key is test_row_0/B:col10/1730993050951/Put/seqid=0 2024-11-07T15:24:12,952 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110797fdc782f46448f09524d54c090ad913_a3486aab3496e5af6c3ff570f5d35d09 store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:12,954 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110797fdc782f46448f09524d54c090ad913_a3486aab3496e5af6c3ff570f5d35d09, store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:12,954 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110797fdc782f46448f09524d54c090ad913_a3486aab3496e5af6c3ff570f5d35d09 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:12,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:24:12,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,962 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing a3486aab3496e5af6c3ff570f5d35d09 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T15:24:12,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742206_1382 (size=4469) 2024-11-07T15:24:12,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742205_1381 (size=12409) 2024-11-07T15:24:12,971 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#A#compaction#319 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:12,972 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/00fdfed8a0bf45979ad3f31beb4a689c is 175, key is test_row_0/A:col10/1730993050951/Put/seqid=0 2024-11-07T15:24:12,974 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/e2753bcb363f48f28f97f87acf346bf7 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/e2753bcb363f48f28f97f87acf346bf7 2024-11-07T15:24:12,978 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/B of a3486aab3496e5af6c3ff570f5d35d09 into e2753bcb363f48f28f97f87acf346bf7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:12,978 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:12,978 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/B, priority=13, startTime=1730993052934; duration=0sec 2024-11-07T15:24:12,978 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:12,978 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:B 2024-11-07T15:24:12,978 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:12,979 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:12,979 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/C is initiating minor compaction (all files) 2024-11-07T15:24:12,979 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/C in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:12,979 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/6c7aeab6003f4c08bc94d0fe7c06f5b2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/5d7ce89540ad4591a3d612c64dac0575, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/433dfe8c88ed46f5acf219d8851a3ce4] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=35.5 K 2024-11-07T15:24:12,979 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c7aeab6003f4c08bc94d0fe7c06f5b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730993048527 2024-11-07T15:24:12,979 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d7ce89540ad4591a3d612c64dac0575, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1730993049737 2024-11-07T15:24:12,980 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 433dfe8c88ed46f5acf219d8851a3ce4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1730993050948 2024-11-07T15:24:12,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079f65e23737d9403d8accc5793a6307fa_a3486aab3496e5af6c3ff570f5d35d09 is 50, key is test_row_0/A:col10/1730993051644/Put/seqid=0 2024-11-07T15:24:12,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742207_1383 (size=31363) 2024-11-07T15:24:12,993 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#C#compaction#322 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:12,994 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/ca85e46154c74190a570b9e71f56268e is 50, key is test_row_0/C:col10/1730993050951/Put/seqid=0 2024-11-07T15:24:13,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T15:24:13,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742208_1384 (size=12304) 2024-11-07T15:24:13,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742209_1385 (size=12409) 2024-11-07T15:24:13,400 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/00fdfed8a0bf45979ad3f31beb4a689c as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/00fdfed8a0bf45979ad3f31beb4a689c 2024-11-07T15:24:13,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:13,406 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/A of a3486aab3496e5af6c3ff570f5d35d09 into 00fdfed8a0bf45979ad3f31beb4a689c(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:13,406 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:13,406 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/A, priority=13, startTime=1730993052934; duration=0sec 2024-11-07T15:24:13,406 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:13,406 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:A 2024-11-07T15:24:13,409 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079f65e23737d9403d8accc5793a6307fa_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079f65e23737d9403d8accc5793a6307fa_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:13,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/71c6014b3166459ba31a25adc2a6c7a0, store: [table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:13,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/71c6014b3166459ba31a25adc2a6c7a0 is 175, key is test_row_0/A:col10/1730993051644/Put/seqid=0 2024-11-07T15:24:13,416 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/ca85e46154c74190a570b9e71f56268e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/ca85e46154c74190a570b9e71f56268e 2024-11-07T15:24:13,421 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/C of a3486aab3496e5af6c3ff570f5d35d09 into ca85e46154c74190a570b9e71f56268e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:13,421 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:13,421 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/C, priority=13, startTime=1730993052934; duration=0sec 2024-11-07T15:24:13,421 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:13,421 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:C 2024-11-07T15:24:13,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742210_1386 (size=31105) 2024-11-07T15:24:13,428 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/71c6014b3166459ba31a25adc2a6c7a0 2024-11-07T15:24:13,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/b50ac70c89cd471a819ece16b5918497 is 50, key is test_row_0/B:col10/1730993051644/Put/seqid=0 2024-11-07T15:24:13,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742211_1387 (size=12151) 2024-11-07T15:24:13,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:13,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:13,845 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/b50ac70c89cd471a819ece16b5918497 2024-11-07T15:24:13,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/e825be2599374818ba7c335531c0e56f is 50, key is test_row_0/C:col10/1730993051644/Put/seqid=0 2024-11-07T15:24:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993113858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:13,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993113858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:13,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993113860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:13,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742212_1388 (size=12151) 2024-11-07T15:24:13,870 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/e825be2599374818ba7c335531c0e56f 2024-11-07T15:24:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:13,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/71c6014b3166459ba31a25adc2a6c7a0 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/71c6014b3166459ba31a25adc2a6c7a0 2024-11-07T15:24:13,877 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/71c6014b3166459ba31a25adc2a6c7a0, entries=150, sequenceid=154, filesize=30.4 K 2024-11-07T15:24:13,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/b50ac70c89cd471a819ece16b5918497 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/b50ac70c89cd471a819ece16b5918497 2024-11-07T15:24:13,882 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/b50ac70c89cd471a819ece16b5918497, entries=150, sequenceid=154, filesize=11.9 K 2024-11-07T15:24:13,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/e825be2599374818ba7c335531c0e56f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/e825be2599374818ba7c335531c0e56f 2024-11-07T15:24:13,886 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/e825be2599374818ba7c335531c0e56f, entries=150, sequenceid=154, filesize=11.9 K 2024-11-07T15:24:13,887 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for a3486aab3496e5af6c3ff570f5d35d09 in 925ms, sequenceid=154, compaction requested=false 2024-11-07T15:24:13,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:13,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:13,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-07T15:24:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-07T15:24:13,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-07T15:24:13,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0270 sec 2024-11-07T15:24:13,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.0310 sec 2024-11-07T15:24:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:13,974 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:13,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3486aab3496e5af6c3ff570f5d35d09 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T15:24:13,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d6fe36af4ef7449f81265941ea335250_a3486aab3496e5af6c3ff570f5d35d09 is 50, key is test_row_0/A:col10/1730993053857/Put/seqid=0 2024-11-07T15:24:13,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742213_1389 (size=17284) 2024-11-07T15:24:13,987 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:13,990 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d6fe36af4ef7449f81265941ea335250_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d6fe36af4ef7449f81265941ea335250_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:13,991 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/c44837fa4bf745608b1bd2758025dc18, store: [table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:13,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/c44837fa4bf745608b1bd2758025dc18 is 175, key is test_row_0/A:col10/1730993053857/Put/seqid=0 2024-11-07T15:24:13,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742214_1390 (size=48389) 2024-11-07T15:24:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T15:24:14,011 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:14,011 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-07T15:24:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T15:24:14,012 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:14,013 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:14,013 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:14,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993114012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:14,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:14,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993114013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:14,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:14,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993114016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:14,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T15:24:14,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993114130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:14,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:14,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993114131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:14,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993114131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:14,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:24:14,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:14,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:14,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:14,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:24:14,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:14,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:14,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:14,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T15:24:14,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:14,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993114340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:14,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993114340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:14,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993114341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:14,397 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/c44837fa4bf745608b1bd2758025dc18 2024-11-07T15:24:14,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/ec0c3c1e3f6d41b3a59154714e5efcac is 50, key is test_row_0/B:col10/1730993053857/Put/seqid=0 2024-11-07T15:24:14,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742215_1391 (size=12151) 2024-11-07T15:24:14,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:24:14,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:14,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:14,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:14,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:24:14,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:14,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:14,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:14,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T15:24:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993114651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993114651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993114651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:14,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:24:14,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:14,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:14,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:14,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/ec0c3c1e3f6d41b3a59154714e5efcac 2024-11-07T15:24:14,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/b9e407999c9a4ee1978cf8b13fe60a14 is 50, key is test_row_0/C:col10/1730993053857/Put/seqid=0 2024-11-07T15:24:14,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742216_1392 (size=12151) 2024-11-07T15:24:14,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:24:14,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:14,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:14,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:14,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:15,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:24:15,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:15,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:15,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:15,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:15,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T15:24:15,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993115160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:15,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:15,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:15,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:15,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993115170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:15,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:15,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:15,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993115171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:15,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:15,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:15,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:15,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:15,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/b9e407999c9a4ee1978cf8b13fe60a14 2024-11-07T15:24:15,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/c44837fa4bf745608b1bd2758025dc18 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/c44837fa4bf745608b1bd2758025dc18 2024-11-07T15:24:15,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/c44837fa4bf745608b1bd2758025dc18, entries=250, sequenceid=176, filesize=47.3 K 2024-11-07T15:24:15,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/ec0c3c1e3f6d41b3a59154714e5efcac as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/ec0c3c1e3f6d41b3a59154714e5efcac 2024-11-07T15:24:15,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/ec0c3c1e3f6d41b3a59154714e5efcac, entries=150, sequenceid=176, filesize=11.9 K 2024-11-07T15:24:15,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/b9e407999c9a4ee1978cf8b13fe60a14 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/b9e407999c9a4ee1978cf8b13fe60a14 2024-11-07T15:24:15,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/b9e407999c9a4ee1978cf8b13fe60a14, entries=150, sequenceid=176, filesize=11.9 K 2024-11-07T15:24:15,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for a3486aab3496e5af6c3ff570f5d35d09 in 1258ms, sequenceid=176, compaction requested=true 2024-11-07T15:24:15,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:15,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:24:15,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:15,233 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:15,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:24:15,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:15,233 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:15,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:24:15,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:15,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:24:15,236 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:15,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:15,236 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/B is initiating minor compaction (all files) 2024-11-07T15:24:15,236 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/B in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:15,236 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing a3486aab3496e5af6c3ff570f5d35d09 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-07T15:24:15,236 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/e2753bcb363f48f28f97f87acf346bf7, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/b50ac70c89cd471a819ece16b5918497, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/ec0c3c1e3f6d41b3a59154714e5efcac] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=35.9 K 2024-11-07T15:24:15,236 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:15,236 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/A is initiating minor compaction (all files) 2024-11-07T15:24:15,236 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/A in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:15,237 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/00fdfed8a0bf45979ad3f31beb4a689c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/71c6014b3166459ba31a25adc2a6c7a0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/c44837fa4bf745608b1bd2758025dc18] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=108.3 K 2024-11-07T15:24:15,237 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:15,237 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/00fdfed8a0bf45979ad3f31beb4a689c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/71c6014b3166459ba31a25adc2a6c7a0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/c44837fa4bf745608b1bd2758025dc18] 2024-11-07T15:24:15,238 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting e2753bcb363f48f28f97f87acf346bf7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1730993050948 2024-11-07T15:24:15,238 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00fdfed8a0bf45979ad3f31beb4a689c, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1730993050948 2024-11-07T15:24:15,238 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting b50ac70c89cd471a819ece16b5918497, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1730993051629 2024-11-07T15:24:15,238 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71c6014b3166459ba31a25adc2a6c7a0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1730993051629 2024-11-07T15:24:15,238 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting ec0c3c1e3f6d41b3a59154714e5efcac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1730993053857 2024-11-07T15:24:15,238 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting c44837fa4bf745608b1bd2758025dc18, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1730993053842 2024-11-07T15:24:15,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a486fba915374e9482dee7b84e9d92ef_a3486aab3496e5af6c3ff570f5d35d09 is 50, key is test_row_0/A:col10/1730993054001/Put/seqid=0 2024-11-07T15:24:15,248 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#B#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:15,248 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:15,249 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/e286f9696e5f434eb7d4bd1e507a27e9 is 50, key is test_row_0/B:col10/1730993053857/Put/seqid=0 2024-11-07T15:24:15,254 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107561b91eed4924775b54fb13e5aca1a72_a3486aab3496e5af6c3ff570f5d35d09 store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:15,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742217_1393 (size=12304) 2024-11-07T15:24:15,255 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107561b91eed4924775b54fb13e5aca1a72_a3486aab3496e5af6c3ff570f5d35d09, store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:15,256 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107561b91eed4924775b54fb13e5aca1a72_a3486aab3496e5af6c3ff570f5d35d09 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:15,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742218_1394 (size=12561) 2024-11-07T15:24:15,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742219_1395 (size=4469) 2024-11-07T15:24:15,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:15,658 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a486fba915374e9482dee7b84e9d92ef_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a486fba915374e9482dee7b84e9d92ef_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:15,663 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/e286f9696e5f434eb7d4bd1e507a27e9 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/e286f9696e5f434eb7d4bd1e507a27e9 2024-11-07T15:24:15,667 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/B of a3486aab3496e5af6c3ff570f5d35d09 into e286f9696e5f434eb7d4bd1e507a27e9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:15,667 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:15,667 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/B, priority=13, startTime=1730993055233; duration=0sec 2024-11-07T15:24:15,667 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:15,667 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:B 2024-11-07T15:24:15,667 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:15,668 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:15,668 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/C is initiating minor compaction (all files) 2024-11-07T15:24:15,668 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/C in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:15,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/8d1652de436d4911bc1a318bfc0531f5, store: [table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:15,668 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/ca85e46154c74190a570b9e71f56268e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/e825be2599374818ba7c335531c0e56f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/b9e407999c9a4ee1978cf8b13fe60a14] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=35.9 K 2024-11-07T15:24:15,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/8d1652de436d4911bc1a318bfc0531f5 is 175, key is test_row_0/A:col10/1730993054001/Put/seqid=0 2024-11-07T15:24:15,669 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting ca85e46154c74190a570b9e71f56268e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1730993050948 2024-11-07T15:24:15,669 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#A#compaction#330 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:15,669 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting e825be2599374818ba7c335531c0e56f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1730993051629 2024-11-07T15:24:15,669 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/4ef72f7d679c44a6bd2e6e77b000345a is 175, key is test_row_0/A:col10/1730993053857/Put/seqid=0 2024-11-07T15:24:15,670 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting b9e407999c9a4ee1978cf8b13fe60a14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1730993053857 2024-11-07T15:24:15,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742220_1396 (size=31105) 2024-11-07T15:24:15,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742221_1397 (size=31515) 2024-11-07T15:24:15,674 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/8d1652de436d4911bc1a318bfc0531f5 2024-11-07T15:24:15,678 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#C#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:15,678 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/bfe71b44601f4e51be7658cb44668d0f is 50, key is test_row_0/C:col10/1730993053857/Put/seqid=0 2024-11-07T15:24:15,679 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/4ef72f7d679c44a6bd2e6e77b000345a as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/4ef72f7d679c44a6bd2e6e77b000345a 2024-11-07T15:24:15,683 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/A of a3486aab3496e5af6c3ff570f5d35d09 into 4ef72f7d679c44a6bd2e6e77b000345a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:15,684 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:15,684 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/A, priority=13, startTime=1730993055232; duration=0sec 2024-11-07T15:24:15,684 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:15,684 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:A 2024-11-07T15:24:15,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/7fbad5457fa04aaebac7083913887032 is 50, key is test_row_0/B:col10/1730993054001/Put/seqid=0 2024-11-07T15:24:15,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742222_1398 (size=12561) 2024-11-07T15:24:15,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742223_1399 (size=12151) 2024-11-07T15:24:15,697 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/bfe71b44601f4e51be7658cb44668d0f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/bfe71b44601f4e51be7658cb44668d0f 2024-11-07T15:24:15,701 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/C of a3486aab3496e5af6c3ff570f5d35d09 into bfe71b44601f4e51be7658cb44668d0f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:15,701 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:15,701 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/C, priority=13, startTime=1730993055233; duration=0sec 2024-11-07T15:24:15,701 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:15,701 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:C 2024-11-07T15:24:16,094 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/7fbad5457fa04aaebac7083913887032 2024-11-07T15:24:16,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/67a9e1cea0ea490b9f93cceedcc1f005 is 50, key is test_row_0/C:col10/1730993054001/Put/seqid=0 2024-11-07T15:24:16,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742224_1400 (size=12151) 2024-11-07T15:24:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T15:24:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:16,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:16,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993116221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:16,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993116224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993116225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:16,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993116340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993116340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:16,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993116340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:16,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:16,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:16,503 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/67a9e1cea0ea490b9f93cceedcc1f005 2024-11-07T15:24:16,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/8d1652de436d4911bc1a318bfc0531f5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/8d1652de436d4911bc1a318bfc0531f5 2024-11-07T15:24:16,509 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/8d1652de436d4911bc1a318bfc0531f5, entries=150, sequenceid=193, filesize=30.4 K 2024-11-07T15:24:16,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/7fbad5457fa04aaebac7083913887032 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/7fbad5457fa04aaebac7083913887032 2024-11-07T15:24:16,512 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/7fbad5457fa04aaebac7083913887032, entries=150, sequenceid=193, filesize=11.9 K 2024-11-07T15:24:16,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/67a9e1cea0ea490b9f93cceedcc1f005 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/67a9e1cea0ea490b9f93cceedcc1f005 2024-11-07T15:24:16,516 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/67a9e1cea0ea490b9f93cceedcc1f005, entries=150, sequenceid=193, filesize=11.9 K 2024-11-07T15:24:16,517 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a3486aab3496e5af6c3ff570f5d35d09 in 1280ms, sequenceid=193, compaction requested=false 2024-11-07T15:24:16,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:16,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:16,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-07T15:24:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-07T15:24:16,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-07T15:24:16,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5050 sec 2024-11-07T15:24:16,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.5090 sec 2024-11-07T15:24:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:16,555 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:16,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3486aab3496e5af6c3ff570f5d35d09 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T15:24:16,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107aac70d9b8bf64747aeddbc5195735b62_a3486aab3496e5af6c3ff570f5d35d09 is 50, key is test_row_0/A:col10/1730993056224/Put/seqid=0 2024-11-07T15:24:16,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742225_1401 (size=17284) 2024-11-07T15:24:16,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993116595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:16,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993116600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:16,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993116602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:16,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:16,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:16,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993116711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:16,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993116711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993116711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:16,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993116931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:16,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993116931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993116931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:16,978 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:16,981 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107aac70d9b8bf64747aeddbc5195735b62_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107aac70d9b8bf64747aeddbc5195735b62_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:16,981 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/46c2a6a2e5f54575a876a916f5d3252b, store: [table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:16,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/46c2a6a2e5f54575a876a916f5d3252b is 175, key is test_row_0/A:col10/1730993056224/Put/seqid=0 2024-11-07T15:24:16,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742226_1402 (size=48389) 2024-11-07T15:24:17,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993117240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:17,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993117251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:17,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993117251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:17,386 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/46c2a6a2e5f54575a876a916f5d3252b 2024-11-07T15:24:17,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/d9911218dd1e49759d3bb3aab21d7ee1 is 50, key is test_row_0/B:col10/1730993056224/Put/seqid=0 2024-11-07T15:24:17,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742227_1403 (size=12151) 2024-11-07T15:24:17,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993117760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:17,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:17,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993117760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:17,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:17,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:17,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:17,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993117771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:17,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:17,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:17,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:17,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/d9911218dd1e49759d3bb3aab21d7ee1 2024-11-07T15:24:17,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/59c43022ba654e1db838cab4d7fdcc41 is 50, key is test_row_0/C:col10/1730993056224/Put/seqid=0 2024-11-07T15:24:17,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742228_1404 (size=12151) 2024-11-07T15:24:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T15:24:18,181 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:18,181 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-07T15:24:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T15:24:18,183 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:18,184 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:18,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:18,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/59c43022ba654e1db838cab4d7fdcc41 2024-11-07T15:24:18,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/46c2a6a2e5f54575a876a916f5d3252b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/46c2a6a2e5f54575a876a916f5d3252b 2024-11-07T15:24:18,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/46c2a6a2e5f54575a876a916f5d3252b, entries=250, sequenceid=217, filesize=47.3 K 2024-11-07T15:24:18,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/d9911218dd1e49759d3bb3aab21d7ee1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/d9911218dd1e49759d3bb3aab21d7ee1 2024-11-07T15:24:18,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/d9911218dd1e49759d3bb3aab21d7ee1, entries=150, sequenceid=217, filesize=11.9 K 2024-11-07T15:24:18,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/59c43022ba654e1db838cab4d7fdcc41 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/59c43022ba654e1db838cab4d7fdcc41 2024-11-07T15:24:18,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/59c43022ba654e1db838cab4d7fdcc41, entries=150, sequenceid=217, filesize=11.9 K 2024-11-07T15:24:18,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for a3486aab3496e5af6c3ff570f5d35d09 in 1670ms, sequenceid=217, compaction requested=true 2024-11-07T15:24:18,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:18,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:24:18,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:18,225 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:18,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:24:18,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:18,225 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:18,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:24:18,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:18,226 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:18,226 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:18,226 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/A is initiating minor compaction (all files) 2024-11-07T15:24:18,226 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/B is initiating minor compaction (all files) 2024-11-07T15:24:18,227 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/B in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:18,227 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/A in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:18,227 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/e286f9696e5f434eb7d4bd1e507a27e9, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/7fbad5457fa04aaebac7083913887032, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/d9911218dd1e49759d3bb3aab21d7ee1] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=36.0 K 2024-11-07T15:24:18,227 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/4ef72f7d679c44a6bd2e6e77b000345a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/8d1652de436d4911bc1a318bfc0531f5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/46c2a6a2e5f54575a876a916f5d3252b] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=108.4 K 2024-11-07T15:24:18,227 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:18,227 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/4ef72f7d679c44a6bd2e6e77b000345a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/8d1652de436d4911bc1a318bfc0531f5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/46c2a6a2e5f54575a876a916f5d3252b] 2024-11-07T15:24:18,227 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting e286f9696e5f434eb7d4bd1e507a27e9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1730993053857 2024-11-07T15:24:18,227 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ef72f7d679c44a6bd2e6e77b000345a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1730993053857 2024-11-07T15:24:18,227 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fbad5457fa04aaebac7083913887032, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1730993054001 2024-11-07T15:24:18,227 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d1652de436d4911bc1a318bfc0531f5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1730993054001 2024-11-07T15:24:18,227 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting d9911218dd1e49759d3bb3aab21d7ee1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1730993056224 2024-11-07T15:24:18,227 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46c2a6a2e5f54575a876a916f5d3252b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1730993056205 2024-11-07T15:24:18,234 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#B#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:18,234 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/f70cfb7b710c4a4aa06c68596112f70c is 50, key is test_row_0/B:col10/1730993056224/Put/seqid=0 2024-11-07T15:24:18,235 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:18,245 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411070250567b42004a46bdb1fae64b922220_a3486aab3496e5af6c3ff570f5d35d09 store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:18,247 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411070250567b42004a46bdb1fae64b922220_a3486aab3496e5af6c3ff570f5d35d09, store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:18,247 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070250567b42004a46bdb1fae64b922220_a3486aab3496e5af6c3ff570f5d35d09 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:18,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742229_1405 (size=12663) 2024-11-07T15:24:18,256 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/f70cfb7b710c4a4aa06c68596112f70c as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/f70cfb7b710c4a4aa06c68596112f70c 2024-11-07T15:24:18,261 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/B of a3486aab3496e5af6c3ff570f5d35d09 into f70cfb7b710c4a4aa06c68596112f70c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:18,261 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:18,261 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/B, priority=13, startTime=1730993058225; duration=0sec 2024-11-07T15:24:18,261 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:18,261 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:B 2024-11-07T15:24:18,261 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:18,262 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:18,262 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/C is initiating minor compaction (all files) 2024-11-07T15:24:18,262 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/C in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:18,262 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/bfe71b44601f4e51be7658cb44668d0f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/67a9e1cea0ea490b9f93cceedcc1f005, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/59c43022ba654e1db838cab4d7fdcc41] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=36.0 K 2024-11-07T15:24:18,263 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting bfe71b44601f4e51be7658cb44668d0f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1730993053857 2024-11-07T15:24:18,263 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 67a9e1cea0ea490b9f93cceedcc1f005, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1730993054001 2024-11-07T15:24:18,263 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 59c43022ba654e1db838cab4d7fdcc41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1730993056224 2024-11-07T15:24:18,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742230_1406 (size=4469) 2024-11-07T15:24:18,285 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#C#compaction#339 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:18,285 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/14ccab2318494d1cbb13ac072e40bd7a is 50, key is test_row_0/C:col10/1730993056224/Put/seqid=0 2024-11-07T15:24:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T15:24:18,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742231_1407 (size=12663) 2024-11-07T15:24:18,306 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/14ccab2318494d1cbb13ac072e40bd7a as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/14ccab2318494d1cbb13ac072e40bd7a 2024-11-07T15:24:18,314 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/C of a3486aab3496e5af6c3ff570f5d35d09 into 14ccab2318494d1cbb13ac072e40bd7a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:18,314 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:18,314 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/C, priority=13, startTime=1730993058225; duration=0sec 2024-11-07T15:24:18,314 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:18,314 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:C 2024-11-07T15:24:18,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T15:24:18,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:18,336 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing a3486aab3496e5af6c3ff570f5d35d09 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T15:24:18,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079d263b489d3b4d03a8d3379452598c3c_a3486aab3496e5af6c3ff570f5d35d09 is 50, key is test_row_0/A:col10/1730993056594/Put/seqid=0 2024-11-07T15:24:18,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742232_1408 (size=12304) 2024-11-07T15:24:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T15:24:18,681 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#A#compaction#338 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:18,681 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/c27dd553fa1647a29abeec6b655c9a58 is 175, key is test_row_0/A:col10/1730993056224/Put/seqid=0 2024-11-07T15:24:18,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742233_1409 (size=31617) 2024-11-07T15:24:18,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:18,763 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079d263b489d3b4d03a8d3379452598c3c_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079d263b489d3b4d03a8d3379452598c3c_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:18,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/8142413187684972b92e7cd92c53acb3, store: [table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:18,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/8142413187684972b92e7cd92c53acb3 is 175, key is test_row_0/A:col10/1730993056594/Put/seqid=0 2024-11-07T15:24:18,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742234_1410 (size=31105) 2024-11-07T15:24:18,769 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/8142413187684972b92e7cd92c53acb3 2024-11-07T15:24:18,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/90f3275db7b14b4b822d44c67cf140f6 is 50, key is test_row_0/B:col10/1730993056594/Put/seqid=0 2024-11-07T15:24:18,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742235_1411 (size=12151) 2024-11-07T15:24:18,781 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/90f3275db7b14b4b822d44c67cf140f6 2024-11-07T15:24:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:18,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:18,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/df69f2710b3b4d4eb597a6456271c9fc is 50, key is test_row_0/C:col10/1730993056594/Put/seqid=0 2024-11-07T15:24:18,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742236_1412 (size=12151) 2024-11-07T15:24:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T15:24:18,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993118827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:18,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993118832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:18,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993118833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:18,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993118941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:18,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:18,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993118950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:18,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993118950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:19,090 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/c27dd553fa1647a29abeec6b655c9a58 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/c27dd553fa1647a29abeec6b655c9a58 2024-11-07T15:24:19,095 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/A of a3486aab3496e5af6c3ff570f5d35d09 into c27dd553fa1647a29abeec6b655c9a58(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:19,095 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:19,095 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/A, priority=13, startTime=1730993058225; duration=0sec 2024-11-07T15:24:19,095 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:19,095 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:A 2024-11-07T15:24:19,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993119151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:19,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993119170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:19,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993119170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:19,197 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/df69f2710b3b4d4eb597a6456271c9fc 2024-11-07T15:24:19,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/8142413187684972b92e7cd92c53acb3 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/8142413187684972b92e7cd92c53acb3 2024-11-07T15:24:19,203 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/8142413187684972b92e7cd92c53acb3, entries=150, sequenceid=234, filesize=30.4 K 2024-11-07T15:24:19,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/90f3275db7b14b4b822d44c67cf140f6 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/90f3275db7b14b4b822d44c67cf140f6 2024-11-07T15:24:19,209 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/90f3275db7b14b4b822d44c67cf140f6, entries=150, sequenceid=234, filesize=11.9 K 2024-11-07T15:24:19,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/df69f2710b3b4d4eb597a6456271c9fc as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/df69f2710b3b4d4eb597a6456271c9fc 2024-11-07T15:24:19,213 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/df69f2710b3b4d4eb597a6456271c9fc, entries=150, sequenceid=234, filesize=11.9 K 2024-11-07T15:24:19,214 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for a3486aab3496e5af6c3ff570f5d35d09 in 878ms, sequenceid=234, compaction requested=false 2024-11-07T15:24:19,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:19,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:19,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-07T15:24:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-07T15:24:19,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-07T15:24:19,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0320 sec 2024-11-07T15:24:19,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.0370 sec 2024-11-07T15:24:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T15:24:19,321 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:19,321 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-07T15:24:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T15:24:19,322 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:19,323 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:19,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T15:24:19,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T15:24:19,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:19,475 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing a3486aab3496e5af6c3ff570f5d35d09 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T15:24:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:19,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:19,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073dc876aa4f244a1693c75c6774590ab1_a3486aab3496e5af6c3ff570f5d35d09 is 50, key is test_row_0/A:col10/1730993058831/Put/seqid=0 2024-11-07T15:24:19,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742237_1413 (size=14844) 2024-11-07T15:24:19,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:19,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993119505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:19,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:19,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993119512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:19,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:19,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:19,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993119515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:19,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:19,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993119620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:19,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993119630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:19,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993119630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T15:24:19,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993119830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:19,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:19,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993119851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:19,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:19,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993119851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:19,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:19,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:19,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:19,889 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073dc876aa4f244a1693c75c6774590ab1_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073dc876aa4f244a1693c75c6774590ab1_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:19,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/7679487581da4adcbf7e2c34c77e5708, store: [table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:19,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/7679487581da4adcbf7e2c34c77e5708 is 175, key is test_row_0/A:col10/1730993058831/Put/seqid=0 2024-11-07T15:24:19,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742238_1414 (size=39799) 2024-11-07T15:24:19,893 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/7679487581da4adcbf7e2c34c77e5708 2024-11-07T15:24:19,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/72ea5e88040743aeb4412bd2f80cfdf8 is 50, key is test_row_0/B:col10/1730993058831/Put/seqid=0 2024-11-07T15:24:19,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742239_1415 (size=12201) 2024-11-07T15:24:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T15:24:20,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993120151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:20,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993120160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:20,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993120160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:20,305 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/72ea5e88040743aeb4412bd2f80cfdf8 2024-11-07T15:24:20,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/c624066bc4b64a1d99f1954d37a7c782 is 50, key is test_row_0/C:col10/1730993058831/Put/seqid=0 2024-11-07T15:24:20,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742240_1416 (size=12201) 2024-11-07T15:24:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T15:24:20,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:20,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993120660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:20,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:20,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:20,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:20,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993120671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:20,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:20,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993120671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:20,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:20,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:20,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:20,718 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/c624066bc4b64a1d99f1954d37a7c782 2024-11-07T15:24:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/7679487581da4adcbf7e2c34c77e5708 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/7679487581da4adcbf7e2c34c77e5708 2024-11-07T15:24:20,725 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/7679487581da4adcbf7e2c34c77e5708, entries=200, sequenceid=257, filesize=38.9 K 2024-11-07T15:24:20,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/72ea5e88040743aeb4412bd2f80cfdf8 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/72ea5e88040743aeb4412bd2f80cfdf8 2024-11-07T15:24:20,729 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/72ea5e88040743aeb4412bd2f80cfdf8, entries=150, sequenceid=257, filesize=11.9 K 2024-11-07T15:24:20,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/c624066bc4b64a1d99f1954d37a7c782 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/c624066bc4b64a1d99f1954d37a7c782 2024-11-07T15:24:20,743 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/c624066bc4b64a1d99f1954d37a7c782, entries=150, sequenceid=257, filesize=11.9 K 2024-11-07T15:24:20,744 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for a3486aab3496e5af6c3ff570f5d35d09 in 1269ms, sequenceid=257, compaction requested=true 2024-11-07T15:24:20,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:20,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:20,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-07T15:24:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-07T15:24:20,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-07T15:24:20,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4220 sec 2024-11-07T15:24:20,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.4260 sec 2024-11-07T15:24:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T15:24:21,481 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:21,481 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-07T15:24:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T15:24:21,483 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:21,483 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:21,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T15:24:21,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T15:24:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:21,635 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing a3486aab3496e5af6c3ff570f5d35d09 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T15:24:21,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c7fedc84c1244cdabc32f85cf4ea35aa_a3486aab3496e5af6c3ff570f5d35d09 is 50, key is test_row_0/A:col10/1730993059511/Put/seqid=0 2024-11-07T15:24:21,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742241_1417 (size=12454) 2024-11-07T15:24:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:21,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. as already flushing 2024-11-07T15:24:21,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993121734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:21,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:21,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993121739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:21,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993121739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46110 deadline: 1730993121744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:21,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46108 deadline: 1730993121780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:21,790 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in a3486aab3496e5af6c3ff570f5d35d09 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18323 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T15:24:21,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993121850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:21,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:21,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993121850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:21,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:21,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:21,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993121860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:21,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46110 deadline: 1730993121861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:22,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:22,050 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c7fedc84c1244cdabc32f85cf4ea35aa_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c7fedc84c1244cdabc32f85cf4ea35aa_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:22,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/6fdf8107887046f5b5b97a2a1cc086c1, store: [table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:22,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/6fdf8107887046f5b5b97a2a1cc086c1 is 175, key is test_row_0/A:col10/1730993059511/Put/seqid=0 2024-11-07T15:24:22,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742242_1418 (size=31255) 2024-11-07T15:24:22,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993122061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:22,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:22,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993122061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:22,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993122070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46110 deadline: 1730993122071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T15:24:22,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993122381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:22,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:22,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993122381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:22,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:22,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993122390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:22,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46110 deadline: 1730993122390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:22,455 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/6fdf8107887046f5b5b97a2a1cc086c1 2024-11-07T15:24:22,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/ba022af6cc034ce084f8e3004967b18d is 50, key is test_row_0/B:col10/1730993059511/Put/seqid=0 2024-11-07T15:24:22,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742243_1419 (size=12301) 2024-11-07T15:24:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T15:24:22,871 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/ba022af6cc034ce084f8e3004967b18d 2024-11-07T15:24:22,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/b71e7a604fc74b5db26633ef4770daef is 50, key is test_row_0/C:col10/1730993059511/Put/seqid=0 2024-11-07T15:24:22,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742244_1420 (size=12301) 2024-11-07T15:24:22,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46194 deadline: 1730993122900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:22,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46158 deadline: 1730993122900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:22,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46110 deadline: 1730993122911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46092 deadline: 1730993122911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:24:23,282 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/b71e7a604fc74b5db26633ef4770daef 2024-11-07T15:24:23,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/6fdf8107887046f5b5b97a2a1cc086c1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6fdf8107887046f5b5b97a2a1cc086c1 2024-11-07T15:24:23,288 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6fdf8107887046f5b5b97a2a1cc086c1, entries=150, sequenceid=271, filesize=30.5 K 2024-11-07T15:24:23,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/ba022af6cc034ce084f8e3004967b18d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/ba022af6cc034ce084f8e3004967b18d 2024-11-07T15:24:23,292 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/ba022af6cc034ce084f8e3004967b18d, entries=150, sequenceid=271, filesize=12.0 K 2024-11-07T15:24:23,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/b71e7a604fc74b5db26633ef4770daef as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/b71e7a604fc74b5db26633ef4770daef 2024-11-07T15:24:23,295 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/b71e7a604fc74b5db26633ef4770daef, entries=150, sequenceid=271, filesize=12.0 K 2024-11-07T15:24:23,296 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a3486aab3496e5af6c3ff570f5d35d09 in 1661ms, sequenceid=271, compaction requested=true 2024-11-07T15:24:23,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:23,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:23,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-07T15:24:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-07T15:24:23,298 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-07T15:24:23,298 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8140 sec 2024-11-07T15:24:23,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.8170 sec 2024-11-07T15:24:23,440 INFO [Thread-1574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1574. 2024-11-07T15:24:23,440 DEBUG [Thread-1574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:23,441 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,441 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,441 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:23,442 INFO [Thread-1576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1576. 2024-11-07T15:24:23,442 DEBUG [Thread-1576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:23,442 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,442 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:23,443 INFO [Thread-1580 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1580. 2024-11-07T15:24:23,443 DEBUG [Thread-1580 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:23,443 DEBUG [Thread-1580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,443 DEBUG [Thread-1580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:23,448 INFO [Thread-1582 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1582. 2024-11-07T15:24:23,448 DEBUG [Thread-1582 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:23,448 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,448 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:23,449 INFO [Thread-1578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1578. 2024-11-07T15:24:23,449 DEBUG [Thread-1578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:23,449 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,449 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T15:24:23,631 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:23,914 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:23,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3486aab3496e5af6c3ff570f5d35d09 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T15:24:23,915 INFO [Thread-1563 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1563. 2024-11-07T15:24:23,915 DEBUG [Thread-1563 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:23,915 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,915 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:23,923 INFO [Thread-1571 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1571. 2024-11-07T15:24:23,923 INFO [Thread-1567 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1567. 2024-11-07T15:24:23,923 DEBUG [Thread-1571 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:23,923 DEBUG [Thread-1567 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:23,923 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,923 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,923 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,923 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:23,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:23,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075da68d9f5ab04768b6d602b7459c6311_a3486aab3496e5af6c3ff570f5d35d09 is 50, key is test_row_0/A:col10/1730993061713/Put/seqid=0 2024-11-07T15:24:23,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742245_1421 (size=12454) 2024-11-07T15:24:23,932 INFO [Thread-1565 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1565. 2024-11-07T15:24:23,932 DEBUG [Thread-1565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:23,932 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,932 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:23,932 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:24,329 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:24,335 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075da68d9f5ab04768b6d602b7459c6311_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075da68d9f5ab04768b6d602b7459c6311_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:24,336 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/1423f8b4e6e6481b92a66fbf2d3f5b00, store: [table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:24,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/1423f8b4e6e6481b92a66fbf2d3f5b00 is 175, key is test_row_0/A:col10/1730993061713/Put/seqid=0 2024-11-07T15:24:24,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742246_1422 (size=31255) 2024-11-07T15:24:24,742 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/1423f8b4e6e6481b92a66fbf2d3f5b00 2024-11-07T15:24:24,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/4be7e3b9849543d9ad4f538139dfeab9 is 50, key is test_row_0/B:col10/1730993061713/Put/seqid=0 2024-11-07T15:24:24,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742247_1423 (size=12301) 2024-11-07T15:24:25,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/4be7e3b9849543d9ad4f538139dfeab9 2024-11-07T15:24:25,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/6144430763fc4c818130c43c984580f0 is 50, key is test_row_0/C:col10/1730993061713/Put/seqid=0 2024-11-07T15:24:25,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742248_1424 (size=12301) 2024-11-07T15:24:25,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/6144430763fc4c818130c43c984580f0 2024-11-07T15:24:25,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/1423f8b4e6e6481b92a66fbf2d3f5b00 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/1423f8b4e6e6481b92a66fbf2d3f5b00 2024-11-07T15:24:25,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/1423f8b4e6e6481b92a66fbf2d3f5b00, entries=150, sequenceid=294, filesize=30.5 K 2024-11-07T15:24:25,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/4be7e3b9849543d9ad4f538139dfeab9 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/4be7e3b9849543d9ad4f538139dfeab9 2024-11-07T15:24:25,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/4be7e3b9849543d9ad4f538139dfeab9, entries=150, sequenceid=294, filesize=12.0 K 2024-11-07T15:24:25,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/6144430763fc4c818130c43c984580f0 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/6144430763fc4c818130c43c984580f0 2024-11-07T15:24:25,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/6144430763fc4c818130c43c984580f0, entries=150, sequenceid=294, filesize=12.0 K 2024-11-07T15:24:25,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=20.13 KB/20610 for a3486aab3496e5af6c3ff570f5d35d09 in 1686ms, sequenceid=294, compaction requested=true 2024-11-07T15:24:25,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:25,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:24:25,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:25,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:24:25,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:25,600 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T15:24:25,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3486aab3496e5af6c3ff570f5d35d09:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:24:25,600 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T15:24:25,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:25,601 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61617 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T15:24:25,601 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 165031 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T15:24:25,601 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/A is initiating minor compaction (all files) 2024-11-07T15:24:25,601 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/B is initiating minor compaction (all files) 2024-11-07T15:24:25,601 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/A in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:25,601 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/B in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:25,602 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/f70cfb7b710c4a4aa06c68596112f70c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/90f3275db7b14b4b822d44c67cf140f6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/72ea5e88040743aeb4412bd2f80cfdf8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/ba022af6cc034ce084f8e3004967b18d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/4be7e3b9849543d9ad4f538139dfeab9] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=60.2 K 2024-11-07T15:24:25,602 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/c27dd553fa1647a29abeec6b655c9a58, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/8142413187684972b92e7cd92c53acb3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/7679487581da4adcbf7e2c34c77e5708, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6fdf8107887046f5b5b97a2a1cc086c1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/1423f8b4e6e6481b92a66fbf2d3f5b00] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=161.2 K 2024-11-07T15:24:25,602 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:25,602 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/c27dd553fa1647a29abeec6b655c9a58, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/8142413187684972b92e7cd92c53acb3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/7679487581da4adcbf7e2c34c77e5708, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6fdf8107887046f5b5b97a2a1cc086c1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/1423f8b4e6e6481b92a66fbf2d3f5b00] 2024-11-07T15:24:25,602 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting f70cfb7b710c4a4aa06c68596112f70c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1730993056224 2024-11-07T15:24:25,602 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting c27dd553fa1647a29abeec6b655c9a58, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1730993056224 2024-11-07T15:24:25,602 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 90f3275db7b14b4b822d44c67cf140f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1730993056593 2024-11-07T15:24:25,602 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8142413187684972b92e7cd92c53acb3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1730993056593 2024-11-07T15:24:25,602 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 72ea5e88040743aeb4412bd2f80cfdf8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1730993058801 2024-11-07T15:24:25,602 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7679487581da4adcbf7e2c34c77e5708, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1730993058801 2024-11-07T15:24:25,602 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting ba022af6cc034ce084f8e3004967b18d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1730993059504 2024-11-07T15:24:25,602 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fdf8107887046f5b5b97a2a1cc086c1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1730993059504 2024-11-07T15:24:25,603 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 4be7e3b9849543d9ad4f538139dfeab9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730993061713 2024-11-07T15:24:25,603 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1423f8b4e6e6481b92a66fbf2d3f5b00, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730993061713 2024-11-07T15:24:25,609 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:25,611 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107186fb99ecbbd4ba599471bb0c64baad4_a3486aab3496e5af6c3ff570f5d35d09 store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:25,613 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#B#compaction#352 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:25,613 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/644539b33c894d3f9d605c574ea56b81 is 50, key is test_row_0/B:col10/1730993061713/Put/seqid=0 2024-11-07T15:24:25,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742249_1425 (size=12983) 2024-11-07T15:24:25,637 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107186fb99ecbbd4ba599471bb0c64baad4_a3486aab3496e5af6c3ff570f5d35d09, store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:25,638 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107186fb99ecbbd4ba599471bb0c64baad4_a3486aab3496e5af6c3ff570f5d35d09 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:25,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742250_1426 (size=4469) 2024-11-07T15:24:26,027 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/644539b33c894d3f9d605c574ea56b81 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/644539b33c894d3f9d605c574ea56b81 2024-11-07T15:24:26,033 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/B of a3486aab3496e5af6c3ff570f5d35d09 into 644539b33c894d3f9d605c574ea56b81(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:26,033 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:26,033 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/B, priority=11, startTime=1730993065600; duration=0sec 2024-11-07T15:24:26,033 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:26,033 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:B 2024-11-07T15:24:26,033 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T15:24:26,034 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61617 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T15:24:26,034 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a3486aab3496e5af6c3ff570f5d35d09/C is initiating minor compaction (all files) 2024-11-07T15:24:26,034 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3486aab3496e5af6c3ff570f5d35d09/C in TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:26,034 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/14ccab2318494d1cbb13ac072e40bd7a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/df69f2710b3b4d4eb597a6456271c9fc, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/c624066bc4b64a1d99f1954d37a7c782, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/b71e7a604fc74b5db26633ef4770daef, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/6144430763fc4c818130c43c984580f0] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp, totalSize=60.2 K 2024-11-07T15:24:26,034 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 14ccab2318494d1cbb13ac072e40bd7a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1730993056224 2024-11-07T15:24:26,035 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting df69f2710b3b4d4eb597a6456271c9fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1730993056593 2024-11-07T15:24:26,035 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting c624066bc4b64a1d99f1954d37a7c782, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1730993058801 2024-11-07T15:24:26,035 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting b71e7a604fc74b5db26633ef4770daef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1730993059504 2024-11-07T15:24:26,035 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 6144430763fc4c818130c43c984580f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1730993061713 2024-11-07T15:24:26,042 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#C#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:26,043 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/2dfed9d368914da7989f5818f67cf2f5 is 50, key is test_row_0/C:col10/1730993061713/Put/seqid=0 2024-11-07T15:24:26,044 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3486aab3496e5af6c3ff570f5d35d09#A#compaction#353 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:26,045 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/97bb654ecbc549589766fd8565872daf is 175, key is test_row_0/A:col10/1730993061713/Put/seqid=0 2024-11-07T15:24:26,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742251_1427 (size=12983) 2024-11-07T15:24:26,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742252_1428 (size=31937) 2024-11-07T15:24:26,457 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/97bb654ecbc549589766fd8565872daf as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/97bb654ecbc549589766fd8565872daf 2024-11-07T15:24:26,457 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/2dfed9d368914da7989f5818f67cf2f5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/2dfed9d368914da7989f5818f67cf2f5 2024-11-07T15:24:26,462 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/C of a3486aab3496e5af6c3ff570f5d35d09 into 2dfed9d368914da7989f5818f67cf2f5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:26,462 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a3486aab3496e5af6c3ff570f5d35d09/A of a3486aab3496e5af6c3ff570f5d35d09 into 97bb654ecbc549589766fd8565872daf(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:26,462 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:26,462 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3486aab3496e5af6c3ff570f5d35d09: 2024-11-07T15:24:26,462 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/A, priority=11, startTime=1730993065600; duration=0sec 2024-11-07T15:24:26,462 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., storeName=a3486aab3496e5af6c3ff570f5d35d09/C, priority=11, startTime=1730993065600; duration=0sec 2024-11-07T15:24:26,463 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:26,463 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:C 2024-11-07T15:24:26,463 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:26,463 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3486aab3496e5af6c3ff570f5d35d09:A 2024-11-07T15:24:30,293 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T15:24:31,842 INFO [Thread-1569 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1569. 2024-11-07T15:24:31,842 DEBUG [Thread-1569 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:31,842 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:31,842 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:31,842 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:31,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T15:24:31,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-07T15:24:31,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-07T15:24:31,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-07T15:24:31,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-07T15:24:31,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-07T15:24:31,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T15:24:31,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T15:24:31,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1346 2024-11-07T15:24:31,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4038 rows 2024-11-07T15:24:31,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1344 2024-11-07T15:24:31,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4032 rows 2024-11-07T15:24:31,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1341 2024-11-07T15:24:31,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4023 rows 2024-11-07T15:24:31,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1344 2024-11-07T15:24:31,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4032 rows 2024-11-07T15:24:31,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1346 2024-11-07T15:24:31,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4038 rows 2024-11-07T15:24:31,843 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T15:24:31,843 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:24:31,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:31,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:31,843 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:31,844 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T15:24:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:31,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T15:24:31,847 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993071846"}]},"ts":"1730993071846"} 2024-11-07T15:24:31,848 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T15:24:31,848 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T15:24:31,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:24:31,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3486aab3496e5af6c3ff570f5d35d09, UNASSIGN}] 2024-11-07T15:24:31,851 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3486aab3496e5af6c3ff570f5d35d09, UNASSIGN 2024-11-07T15:24:31,851 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=a3486aab3496e5af6c3ff570f5d35d09, regionState=CLOSING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:31,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3486aab3496e5af6c3ff570f5d35d09, UNASSIGN because future has completed 2024-11-07T15:24:31,853 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:24:31,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:24:31,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T15:24:32,005 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:32,006 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:24:32,006 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing a3486aab3496e5af6c3ff570f5d35d09, disabling compactions & flushes 2024-11-07T15:24:32,006 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:32,006 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:32,006 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. after waiting 0 ms 2024-11-07T15:24:32,006 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:32,006 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing a3486aab3496e5af6c3ff570f5d35d09 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-07T15:24:32,016 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107580c5a5dc9a34d25a7c61dc09bfc41ba_a3486aab3496e5af6c3ff570f5d35d09 is 50, key is test_row_0/A:col10/1730993063931/Put/seqid=0 2024-11-07T15:24:32,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742253_1429 (size=12454) 2024-11-07T15:24:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T15:24:32,421 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:32,433 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107580c5a5dc9a34d25a7c61dc09bfc41ba_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107580c5a5dc9a34d25a7c61dc09bfc41ba_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:32,434 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/538c62034c65416387da0389dca8fe0f, store: [table=TestAcidGuarantees family=A region=a3486aab3496e5af6c3ff570f5d35d09] 2024-11-07T15:24:32,434 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/538c62034c65416387da0389dca8fe0f is 175, key is test_row_0/A:col10/1730993063931/Put/seqid=0 2024-11-07T15:24:32,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742254_1430 (size=31255) 2024-11-07T15:24:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T15:24:32,840 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/538c62034c65416387da0389dca8fe0f 2024-11-07T15:24:32,854 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/0394028e10ed4d13bda7fa38eb24c41f is 50, key is test_row_0/B:col10/1730993063931/Put/seqid=0 2024-11-07T15:24:32,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742255_1431 (size=12301) 2024-11-07T15:24:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T15:24:33,259 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/0394028e10ed4d13bda7fa38eb24c41f 2024-11-07T15:24:33,272 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/854cfb011218489abbc1d93c60782e39 is 50, key is test_row_0/C:col10/1730993063931/Put/seqid=0 2024-11-07T15:24:33,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742256_1432 (size=12301) 2024-11-07T15:24:33,677 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/854cfb011218489abbc1d93c60782e39 2024-11-07T15:24:33,687 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/A/538c62034c65416387da0389dca8fe0f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/538c62034c65416387da0389dca8fe0f 2024-11-07T15:24:33,693 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/538c62034c65416387da0389dca8fe0f, entries=150, sequenceid=304, filesize=30.5 K 2024-11-07T15:24:33,695 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/B/0394028e10ed4d13bda7fa38eb24c41f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/0394028e10ed4d13bda7fa38eb24c41f 2024-11-07T15:24:33,699 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/0394028e10ed4d13bda7fa38eb24c41f, entries=150, sequenceid=304, filesize=12.0 K 2024-11-07T15:24:33,700 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/.tmp/C/854cfb011218489abbc1d93c60782e39 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/854cfb011218489abbc1d93c60782e39 2024-11-07T15:24:33,703 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/854cfb011218489abbc1d93c60782e39, entries=150, sequenceid=304, filesize=12.0 K 2024-11-07T15:24:33,704 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for a3486aab3496e5af6c3ff570f5d35d09 in 1698ms, sequenceid=304, compaction requested=false 2024-11-07T15:24:33,705 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/4aba4a6a91a94446a5bfd17bb94e0979, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6e9b8057026f4bbcb2b85fb787e27d8b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/7e2ecf043aa746d78a507890a4308a16, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/3038ff7e8b7c40fc9ad19c4be02c66bf, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/a6865629762f4206b491d728382f992d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/aff2137755a74eafb86675c6682dae97, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6d41a8f509ad47da8d22deb59d50119a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/540c60ac58b34317a72dbc187e54350f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/2e8b7a2a17a64c5998fba2510786dd77, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/00fdfed8a0bf45979ad3f31beb4a689c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/71c6014b3166459ba31a25adc2a6c7a0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/c44837fa4bf745608b1bd2758025dc18, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/4ef72f7d679c44a6bd2e6e77b000345a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/8d1652de436d4911bc1a318bfc0531f5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/46c2a6a2e5f54575a876a916f5d3252b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/c27dd553fa1647a29abeec6b655c9a58, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/8142413187684972b92e7cd92c53acb3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/7679487581da4adcbf7e2c34c77e5708, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6fdf8107887046f5b5b97a2a1cc086c1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/1423f8b4e6e6481b92a66fbf2d3f5b00] to archive 2024-11-07T15:24:33,705 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:24:33,707 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/4aba4a6a91a94446a5bfd17bb94e0979 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/4aba4a6a91a94446a5bfd17bb94e0979 2024-11-07T15:24:33,709 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6e9b8057026f4bbcb2b85fb787e27d8b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6e9b8057026f4bbcb2b85fb787e27d8b 2024-11-07T15:24:33,710 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/7e2ecf043aa746d78a507890a4308a16 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/7e2ecf043aa746d78a507890a4308a16 2024-11-07T15:24:33,711 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/3038ff7e8b7c40fc9ad19c4be02c66bf to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/3038ff7e8b7c40fc9ad19c4be02c66bf 2024-11-07T15:24:33,713 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/a6865629762f4206b491d728382f992d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/a6865629762f4206b491d728382f992d 2024-11-07T15:24:33,714 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/aff2137755a74eafb86675c6682dae97 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/aff2137755a74eafb86675c6682dae97 2024-11-07T15:24:33,716 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6d41a8f509ad47da8d22deb59d50119a to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6d41a8f509ad47da8d22deb59d50119a 2024-11-07T15:24:33,717 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/540c60ac58b34317a72dbc187e54350f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/540c60ac58b34317a72dbc187e54350f 2024-11-07T15:24:33,719 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/2e8b7a2a17a64c5998fba2510786dd77 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/2e8b7a2a17a64c5998fba2510786dd77 2024-11-07T15:24:33,720 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/00fdfed8a0bf45979ad3f31beb4a689c to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/00fdfed8a0bf45979ad3f31beb4a689c 2024-11-07T15:24:33,722 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/71c6014b3166459ba31a25adc2a6c7a0 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/71c6014b3166459ba31a25adc2a6c7a0 2024-11-07T15:24:33,724 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/c44837fa4bf745608b1bd2758025dc18 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/c44837fa4bf745608b1bd2758025dc18 2024-11-07T15:24:33,726 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/4ef72f7d679c44a6bd2e6e77b000345a to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/4ef72f7d679c44a6bd2e6e77b000345a 2024-11-07T15:24:33,727 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/8d1652de436d4911bc1a318bfc0531f5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/8d1652de436d4911bc1a318bfc0531f5 2024-11-07T15:24:33,729 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/46c2a6a2e5f54575a876a916f5d3252b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/46c2a6a2e5f54575a876a916f5d3252b 2024-11-07T15:24:33,731 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/c27dd553fa1647a29abeec6b655c9a58 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/c27dd553fa1647a29abeec6b655c9a58 2024-11-07T15:24:33,732 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/8142413187684972b92e7cd92c53acb3 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/8142413187684972b92e7cd92c53acb3 2024-11-07T15:24:33,734 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/7679487581da4adcbf7e2c34c77e5708 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/7679487581da4adcbf7e2c34c77e5708 2024-11-07T15:24:33,736 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6fdf8107887046f5b5b97a2a1cc086c1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/6fdf8107887046f5b5b97a2a1cc086c1 2024-11-07T15:24:33,738 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/1423f8b4e6e6481b92a66fbf2d3f5b00 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/1423f8b4e6e6481b92a66fbf2d3f5b00 2024-11-07T15:24:33,740 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/88a8d58857f94bc9ae33e0b787280afd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/c1d5d7d9a2e644f0928c2d681391bade, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/0b9b2f84d40d40e0921b02988fcf066b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/7f36b3fa05b749febdb934ad962a1308, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/77bf5548d09a40b2b7204beb60008922, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/00fb693f37ec4cdeb0831abe1f525aaa, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/0ca097b54c184bc6b5d517ddeaf19e45, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/63c7418b30264a008ef4a3ba6c5d3d3d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/e2753bcb363f48f28f97f87acf346bf7, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/1a508dee6ae94f6680b6876a01674446, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/b50ac70c89cd471a819ece16b5918497, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/e286f9696e5f434eb7d4bd1e507a27e9, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/ec0c3c1e3f6d41b3a59154714e5efcac, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/7fbad5457fa04aaebac7083913887032, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/f70cfb7b710c4a4aa06c68596112f70c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/d9911218dd1e49759d3bb3aab21d7ee1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/90f3275db7b14b4b822d44c67cf140f6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/72ea5e88040743aeb4412bd2f80cfdf8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/ba022af6cc034ce084f8e3004967b18d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/4be7e3b9849543d9ad4f538139dfeab9] to archive 2024-11-07T15:24:33,741 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:24:33,744 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/88a8d58857f94bc9ae33e0b787280afd to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/88a8d58857f94bc9ae33e0b787280afd 2024-11-07T15:24:33,746 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/c1d5d7d9a2e644f0928c2d681391bade to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/c1d5d7d9a2e644f0928c2d681391bade 2024-11-07T15:24:33,748 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/0b9b2f84d40d40e0921b02988fcf066b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/0b9b2f84d40d40e0921b02988fcf066b 2024-11-07T15:24:33,750 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/7f36b3fa05b749febdb934ad962a1308 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/7f36b3fa05b749febdb934ad962a1308 2024-11-07T15:24:33,751 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/77bf5548d09a40b2b7204beb60008922 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/77bf5548d09a40b2b7204beb60008922 2024-11-07T15:24:33,753 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/00fb693f37ec4cdeb0831abe1f525aaa to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/00fb693f37ec4cdeb0831abe1f525aaa 2024-11-07T15:24:33,755 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/0ca097b54c184bc6b5d517ddeaf19e45 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/0ca097b54c184bc6b5d517ddeaf19e45 2024-11-07T15:24:33,757 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/63c7418b30264a008ef4a3ba6c5d3d3d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/63c7418b30264a008ef4a3ba6c5d3d3d 2024-11-07T15:24:33,758 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/e2753bcb363f48f28f97f87acf346bf7 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/e2753bcb363f48f28f97f87acf346bf7 2024-11-07T15:24:33,760 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/1a508dee6ae94f6680b6876a01674446 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/1a508dee6ae94f6680b6876a01674446 2024-11-07T15:24:33,762 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/b50ac70c89cd471a819ece16b5918497 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/b50ac70c89cd471a819ece16b5918497 2024-11-07T15:24:33,764 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/e286f9696e5f434eb7d4bd1e507a27e9 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/e286f9696e5f434eb7d4bd1e507a27e9 2024-11-07T15:24:33,766 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/ec0c3c1e3f6d41b3a59154714e5efcac to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/ec0c3c1e3f6d41b3a59154714e5efcac 2024-11-07T15:24:33,768 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/7fbad5457fa04aaebac7083913887032 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/7fbad5457fa04aaebac7083913887032 2024-11-07T15:24:33,770 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/f70cfb7b710c4a4aa06c68596112f70c to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/f70cfb7b710c4a4aa06c68596112f70c 2024-11-07T15:24:33,772 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/d9911218dd1e49759d3bb3aab21d7ee1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/d9911218dd1e49759d3bb3aab21d7ee1 2024-11-07T15:24:33,774 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/90f3275db7b14b4b822d44c67cf140f6 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/90f3275db7b14b4b822d44c67cf140f6 2024-11-07T15:24:33,776 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/72ea5e88040743aeb4412bd2f80cfdf8 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/72ea5e88040743aeb4412bd2f80cfdf8 2024-11-07T15:24:33,778 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/ba022af6cc034ce084f8e3004967b18d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/ba022af6cc034ce084f8e3004967b18d 2024-11-07T15:24:33,779 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/4be7e3b9849543d9ad4f538139dfeab9 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/4be7e3b9849543d9ad4f538139dfeab9 2024-11-07T15:24:33,781 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/0d1a724e1c164b85b4de0857ce8b9e67, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/7b848316cc0a476ea80de55b78f9c360, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/07b044bc4681437d8ff4931d9bcb5309, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/fce4da5ced6e4604974b3f71df3ed2a4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/38847533e61143f689e8b4c15984d445, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/6c7aeab6003f4c08bc94d0fe7c06f5b2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/4609a090939d4d7180fe7496e9b7bd88, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/5d7ce89540ad4591a3d612c64dac0575, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/ca85e46154c74190a570b9e71f56268e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/433dfe8c88ed46f5acf219d8851a3ce4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/e825be2599374818ba7c335531c0e56f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/bfe71b44601f4e51be7658cb44668d0f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/b9e407999c9a4ee1978cf8b13fe60a14, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/67a9e1cea0ea490b9f93cceedcc1f005, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/14ccab2318494d1cbb13ac072e40bd7a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/59c43022ba654e1db838cab4d7fdcc41, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/df69f2710b3b4d4eb597a6456271c9fc, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/c624066bc4b64a1d99f1954d37a7c782, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/b71e7a604fc74b5db26633ef4770daef, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/6144430763fc4c818130c43c984580f0] to archive 2024-11-07T15:24:33,782 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:24:33,785 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/0d1a724e1c164b85b4de0857ce8b9e67 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/0d1a724e1c164b85b4de0857ce8b9e67 2024-11-07T15:24:33,786 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/7b848316cc0a476ea80de55b78f9c360 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/7b848316cc0a476ea80de55b78f9c360 2024-11-07T15:24:33,788 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/07b044bc4681437d8ff4931d9bcb5309 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/07b044bc4681437d8ff4931d9bcb5309 2024-11-07T15:24:33,790 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/fce4da5ced6e4604974b3f71df3ed2a4 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/fce4da5ced6e4604974b3f71df3ed2a4 2024-11-07T15:24:33,792 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/38847533e61143f689e8b4c15984d445 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/38847533e61143f689e8b4c15984d445 2024-11-07T15:24:33,794 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/6c7aeab6003f4c08bc94d0fe7c06f5b2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/6c7aeab6003f4c08bc94d0fe7c06f5b2 2024-11-07T15:24:33,795 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/4609a090939d4d7180fe7496e9b7bd88 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/4609a090939d4d7180fe7496e9b7bd88 2024-11-07T15:24:33,797 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/5d7ce89540ad4591a3d612c64dac0575 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/5d7ce89540ad4591a3d612c64dac0575 2024-11-07T15:24:33,799 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/ca85e46154c74190a570b9e71f56268e to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/ca85e46154c74190a570b9e71f56268e 2024-11-07T15:24:33,800 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/433dfe8c88ed46f5acf219d8851a3ce4 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/433dfe8c88ed46f5acf219d8851a3ce4 2024-11-07T15:24:33,802 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/e825be2599374818ba7c335531c0e56f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/e825be2599374818ba7c335531c0e56f 2024-11-07T15:24:33,804 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/bfe71b44601f4e51be7658cb44668d0f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/bfe71b44601f4e51be7658cb44668d0f 2024-11-07T15:24:33,805 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/b9e407999c9a4ee1978cf8b13fe60a14 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/b9e407999c9a4ee1978cf8b13fe60a14 2024-11-07T15:24:33,807 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/67a9e1cea0ea490b9f93cceedcc1f005 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/67a9e1cea0ea490b9f93cceedcc1f005 2024-11-07T15:24:33,809 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/14ccab2318494d1cbb13ac072e40bd7a to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/14ccab2318494d1cbb13ac072e40bd7a 2024-11-07T15:24:33,810 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/59c43022ba654e1db838cab4d7fdcc41 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/59c43022ba654e1db838cab4d7fdcc41 2024-11-07T15:24:33,812 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/df69f2710b3b4d4eb597a6456271c9fc to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/df69f2710b3b4d4eb597a6456271c9fc 2024-11-07T15:24:33,814 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/c624066bc4b64a1d99f1954d37a7c782 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/c624066bc4b64a1d99f1954d37a7c782 2024-11-07T15:24:33,815 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/b71e7a604fc74b5db26633ef4770daef to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/b71e7a604fc74b5db26633ef4770daef 2024-11-07T15:24:33,816 DEBUG [StoreCloser-TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/6144430763fc4c818130c43c984580f0 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/6144430763fc4c818130c43c984580f0 2024-11-07T15:24:33,820 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/recovered.edits/307.seqid, newMaxSeqId=307, maxSeqId=4 2024-11-07T15:24:33,820 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. 2024-11-07T15:24:33,821 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for a3486aab3496e5af6c3ff570f5d35d09: Waiting for close lock at 1730993072006Running coprocessor pre-close hooks at 1730993072006Disabling compacts and flushes for region at 1730993072006Disabling writes for close at 1730993072006Obtaining lock to block concurrent updates at 1730993072006Preparing flush snapshotting stores in a3486aab3496e5af6c3ff570f5d35d09 at 1730993072006Finished memstore snapshotting TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1730993072007 (+1 ms)Flushing stores of TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09. at 1730993072007Flushing a3486aab3496e5af6c3ff570f5d35d09/A: creating writer at 1730993072008 (+1 ms)Flushing mob file a3486aab3496e5af6c3ff570f5d35d09/A: appending metadata at 1730993072016 (+8 ms)Flushing mob file a3486aab3496e5af6c3ff570f5d35d09/A: closing flushed file at 1730993072016Flushing a3486aab3496e5af6c3ff570f5d35d09/A: appending metadata at 1730993072434 (+418 ms)Flushing a3486aab3496e5af6c3ff570f5d35d09/A: closing flushed file at 1730993072434Flushing a3486aab3496e5af6c3ff570f5d35d09/B: creating writer at 1730993072850 (+416 ms)Flushing a3486aab3496e5af6c3ff570f5d35d09/B: appending metadata at 1730993072854 (+4 ms)Flushing a3486aab3496e5af6c3ff570f5d35d09/B: closing flushed file at 1730993072854Flushing a3486aab3496e5af6c3ff570f5d35d09/C: creating writer at 1730993073269 (+415 ms)Flushing a3486aab3496e5af6c3ff570f5d35d09/C: appending metadata at 1730993073272 (+3 ms)Flushing a3486aab3496e5af6c3ff570f5d35d09/C: closing flushed file at 1730993073272Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45348751: reopening flushed file at 1730993073685 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3780dc56: reopening flushed file at 1730993073694 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c2cc77c: reopening flushed file at 1730993073699 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for a3486aab3496e5af6c3ff570f5d35d09 in 1698ms, sequenceid=304, compaction requested=false at 1730993073704 (+5 ms)Writing region close event to WAL at 1730993073817 (+113 ms)Running coprocessor post-close hooks at 1730993073820 (+3 ms)Closed at 1730993073820 2024-11-07T15:24:33,822 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:33,822 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=a3486aab3496e5af6c3ff570f5d35d09, regionState=CLOSED 2024-11-07T15:24:33,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:24:33,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-07T15:24:33,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure a3486aab3496e5af6c3ff570f5d35d09, server=7b3e4e8e12ef,33849,1730992951906 in 1.9710 sec 2024-11-07T15:24:33,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-07T15:24:33,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3486aab3496e5af6c3ff570f5d35d09, UNASSIGN in 1.9750 sec 2024-11-07T15:24:33,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-07T15:24:33,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9790 sec 2024-11-07T15:24:33,829 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993073829"}]},"ts":"1730993073829"} 2024-11-07T15:24:33,830 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T15:24:33,830 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T15:24:33,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9870 sec 2024-11-07T15:24:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T15:24:33,992 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:33,993 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T15:24:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:33,997 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T15:24:33,999 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:34,003 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,006 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/recovered.edits] 2024-11-07T15:24:34,009 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/538c62034c65416387da0389dca8fe0f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/538c62034c65416387da0389dca8fe0f 2024-11-07T15:24:34,010 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/97bb654ecbc549589766fd8565872daf to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/A/97bb654ecbc549589766fd8565872daf 2024-11-07T15:24:34,012 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/0394028e10ed4d13bda7fa38eb24c41f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/0394028e10ed4d13bda7fa38eb24c41f 2024-11-07T15:24:34,013 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/644539b33c894d3f9d605c574ea56b81 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/B/644539b33c894d3f9d605c574ea56b81 2024-11-07T15:24:34,014 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/2dfed9d368914da7989f5818f67cf2f5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/2dfed9d368914da7989f5818f67cf2f5 2024-11-07T15:24:34,015 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/854cfb011218489abbc1d93c60782e39 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/C/854cfb011218489abbc1d93c60782e39 2024-11-07T15:24:34,017 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/recovered.edits/307.seqid to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09/recovered.edits/307.seqid 2024-11-07T15:24:34,017 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,018 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T15:24:34,018 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T15:24:34,018 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T15:24:34,020 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070bc312dc6ab246fba69837f460e7a06e_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070bc312dc6ab246fba69837f460e7a06e_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,021 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070da166fa06c649ff95a5bb62484d6437_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070da166fa06c649ff95a5bb62484d6437_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,022 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073dc876aa4f244a1693c75c6774590ab1_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073dc876aa4f244a1693c75c6774590ab1_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,022 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074690ba2daecc4bc78096ceedbf594b84_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074690ba2daecc4bc78096ceedbf594b84_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,023 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107580c5a5dc9a34d25a7c61dc09bfc41ba_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107580c5a5dc9a34d25a7c61dc09bfc41ba_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,024 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075da68d9f5ab04768b6d602b7459c6311_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075da68d9f5ab04768b6d602b7459c6311_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,024 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076e7dd8bb668b41a2843538f5fdb1ff93_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076e7dd8bb668b41a2843538f5fdb1ff93_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,025 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110780646c93ee894de996a687a5bdbd168b_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110780646c93ee894de996a687a5bdbd168b_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,026 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110785283bf9a02d45e98bab4cffcdc86ed2_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110785283bf9a02d45e98bab4cffcdc86ed2_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,026 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079d263b489d3b4d03a8d3379452598c3c_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079d263b489d3b4d03a8d3379452598c3c_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,027 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079f65e23737d9403d8accc5793a6307fa_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079f65e23737d9403d8accc5793a6307fa_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,027 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a486fba915374e9482dee7b84e9d92ef_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a486fba915374e9482dee7b84e9d92ef_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,028 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107aac70d9b8bf64747aeddbc5195735b62_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107aac70d9b8bf64747aeddbc5195735b62_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,029 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c7fedc84c1244cdabc32f85cf4ea35aa_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c7fedc84c1244cdabc32f85cf4ea35aa_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,029 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d6fe36af4ef7449f81265941ea335250_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d6fe36af4ef7449f81265941ea335250_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,030 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f060b40c71de477586dddac3954c26c1_a3486aab3496e5af6c3ff570f5d35d09 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f060b40c71de477586dddac3954c26c1_a3486aab3496e5af6c3ff570f5d35d09 2024-11-07T15:24:34,030 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T15:24:34,032 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:34,034 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T15:24:34,035 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T15:24:34,036 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:34,036 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T15:24:34,036 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730993074036"}]},"ts":"9223372036854775807"} 2024-11-07T15:24:34,037 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T15:24:34,037 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a3486aab3496e5af6c3ff570f5d35d09, NAME => 'TestAcidGuarantees,,1730993040287.a3486aab3496e5af6c3ff570f5d35d09.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T15:24:34,037 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T15:24:34,038 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730993074037"}]},"ts":"9223372036854775807"} 2024-11-07T15:24:34,039 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T15:24:34,039 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:34,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-07T15:24:34,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T15:24:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T15:24:34,112 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:34,127 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237 (was 240), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=438 (was 481), ProcessCount=11 (was 11), AvailableMemoryMB=1250 (was 1283) 2024-11-07T15:24:34,136 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=438, ProcessCount=11, AvailableMemoryMB=1249 2024-11-07T15:24:34,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:24:34,137 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:24:34,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:34,139 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T15:24:34,139 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:34,139 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-07T15:24:34,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-07T15:24:34,140 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T15:24:34,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742257_1433 (size=959) 2024-11-07T15:24:34,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-07T15:24:34,348 WARN [regionserver/7b3e4e8e12ef:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-11-07T15:24:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-07T15:24:34,550 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938 2024-11-07T15:24:34,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742258_1434 (size=53) 2024-11-07T15:24:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-07T15:24:34,960 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:24:34,960 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a4f40fe465370ef7d04dfc7daeebbbb9, disabling compactions & flushes 2024-11-07T15:24:34,960 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:34,960 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:34,960 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. after waiting 0 ms 2024-11-07T15:24:34,960 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:34,960 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:34,960 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a4f40fe465370ef7d04dfc7daeebbbb9: Waiting for close lock at 1730993074960Disabling compacts and flushes for region at 1730993074960Disabling writes for close at 1730993074960Writing region close event to WAL at 1730993074960Closed at 1730993074960 2024-11-07T15:24:34,963 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T15:24:34,963 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730993074963"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730993074963"}]},"ts":"1730993074963"} 2024-11-07T15:24:34,966 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T15:24:34,967 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T15:24:34,968 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993074967"}]},"ts":"1730993074967"} 2024-11-07T15:24:34,970 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T15:24:34,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4f40fe465370ef7d04dfc7daeebbbb9, ASSIGN}] 2024-11-07T15:24:34,972 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4f40fe465370ef7d04dfc7daeebbbb9, ASSIGN 2024-11-07T15:24:34,973 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4f40fe465370ef7d04dfc7daeebbbb9, ASSIGN; state=OFFLINE, location=7b3e4e8e12ef,33849,1730992951906; forceNewPlan=false, retain=false 2024-11-07T15:24:35,124 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=a4f40fe465370ef7d04dfc7daeebbbb9, regionState=OPENING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:35,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4f40fe465370ef7d04dfc7daeebbbb9, ASSIGN because future has completed 2024-11-07T15:24:35,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:24:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-07T15:24:35,289 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:35,289 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:24:35,291 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:35,291 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:24:35,291 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:35,291 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:35,292 INFO [StoreOpener-a4f40fe465370ef7d04dfc7daeebbbb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:35,294 INFO [StoreOpener-a4f40fe465370ef7d04dfc7daeebbbb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4f40fe465370ef7d04dfc7daeebbbb9 columnFamilyName A 2024-11-07T15:24:35,294 DEBUG [StoreOpener-a4f40fe465370ef7d04dfc7daeebbbb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:35,295 INFO [StoreOpener-a4f40fe465370ef7d04dfc7daeebbbb9-1 {}] regionserver.HStore(327): Store=a4f40fe465370ef7d04dfc7daeebbbb9/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:24:35,295 INFO [StoreOpener-a4f40fe465370ef7d04dfc7daeebbbb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:35,296 INFO [StoreOpener-a4f40fe465370ef7d04dfc7daeebbbb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4f40fe465370ef7d04dfc7daeebbbb9 columnFamilyName B 2024-11-07T15:24:35,296 DEBUG [StoreOpener-a4f40fe465370ef7d04dfc7daeebbbb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:35,296 INFO [StoreOpener-a4f40fe465370ef7d04dfc7daeebbbb9-1 {}] regionserver.HStore(327): Store=a4f40fe465370ef7d04dfc7daeebbbb9/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:24:35,297 INFO [StoreOpener-a4f40fe465370ef7d04dfc7daeebbbb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:35,298 INFO [StoreOpener-a4f40fe465370ef7d04dfc7daeebbbb9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4f40fe465370ef7d04dfc7daeebbbb9 columnFamilyName C 2024-11-07T15:24:35,298 DEBUG [StoreOpener-a4f40fe465370ef7d04dfc7daeebbbb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:35,298 INFO [StoreOpener-a4f40fe465370ef7d04dfc7daeebbbb9-1 {}] regionserver.HStore(327): Store=a4f40fe465370ef7d04dfc7daeebbbb9/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:24:35,298 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:35,299 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:35,299 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:35,301 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:35,301 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:35,302 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:24:35,304 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:35,307 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T15:24:35,307 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened a4f40fe465370ef7d04dfc7daeebbbb9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64692560, jitterRate=-0.03600573539733887}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:24:35,307 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:35,309 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for a4f40fe465370ef7d04dfc7daeebbbb9: Running coprocessor pre-open hook at 1730993075291Writing region info on filesystem at 1730993075291Initializing all the Stores at 1730993075292 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993075292Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993075292Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993075292Cleaning up temporary data from old regions at 1730993075301 (+9 ms)Running coprocessor post-open hooks at 1730993075308 (+7 ms)Region opened successfully at 1730993075308 2024-11-07T15:24:35,309 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., pid=120, masterSystemTime=1730993075281 2024-11-07T15:24:35,312 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:35,312 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:35,312 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=a4f40fe465370ef7d04dfc7daeebbbb9, regionState=OPEN, openSeqNum=2, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:35,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:24:35,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-07T15:24:35,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 in 187 msec 2024-11-07T15:24:35,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-07T15:24:35,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4f40fe465370ef7d04dfc7daeebbbb9, ASSIGN in 346 msec 2024-11-07T15:24:35,321 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T15:24:35,321 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993075321"}]},"ts":"1730993075321"} 2024-11-07T15:24:35,323 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T15:24:35,324 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T15:24:35,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1870 sec 2024-11-07T15:24:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-07T15:24:36,303 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:36,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a6d3d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,307 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:36,307 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:36,309 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:36,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:36,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:36,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18a3a102, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:36,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:36,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:36,311 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:36,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d39ae47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,311 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:36,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:36,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:36,313 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:36,314 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T15:24:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b3e4e8e12ef,46811,1730992951165 2024-11-07T15:24:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7ed78e7b 2024-11-07T15:24:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T15:24:36,316 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T15:24:36,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f704f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:36,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:36,318 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b12dc8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:36,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:36,319 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34f49099, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b7894c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:36,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:36,321 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:36,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:36,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:36,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@313bceb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:36,322 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:36,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3199ff22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1447b9d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:36,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:36,324 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:36,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:36,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:36,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b985b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:36,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:36,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:36,325 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c15031e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1de6e594, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:36,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:36,327 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d4679ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:36,328 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@209091a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29cadc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:36,329 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:36,329 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29a31c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:36,331 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:36,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7083d18d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b469189, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:36,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:36,333 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@136f0a29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:36,334 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56dc6086, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6240d801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:36,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:36,336 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae799ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:36,338 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c40bf6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e2d9ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,339 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:36,339 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:36,340 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:36,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:36,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:36,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36bd0eb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:36,342 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:36,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8534bec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8d9f546, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:36,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:36,344 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab5a0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:36,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:36,345 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:36,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e6b9b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d01c47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,346 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:24:36,346 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:24:36,347 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:24:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:24:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:24:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fb75a20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:24:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:24:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:36,348 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:24:36,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c62f242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:24:36,349 DEBUG [Thread-1966 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:36,349 DEBUG [Thread-1951 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:36,349 DEBUG [Thread-1962 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:36,349 DEBUG [Thread-1964 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:36,350 DEBUG [Thread-1953 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:36,350 DEBUG [Thread-1959 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:36,350 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-07T15:24:36,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:36,352 DEBUG [Thread-1968 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:36,352 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:36,352 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:36,353 DEBUG [Thread-1955 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:36,353 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:36,353 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:36,353 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T15:24:36,353 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:36,354 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:36,354 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:36,355 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:24:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:24:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:24:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:24:36,355 DEBUG [Thread-1970 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:36,355 DEBUG [Thread-1957 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:24:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:24:36,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:36,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:24:36,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:36,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:36,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:24:36,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:24:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:24:36,360 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:36,360 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:36,360 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:24:36,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:24:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:24:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2] 2024-11-07T15:24:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:36,367 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:36,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T15:24:36,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/77a9bfbf41134415bafaf4dba9c8c99f is 50, key is test_row_0/A:col10/1730993076366/Put/seqid=0 2024-11-07T15:24:36,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742259_1435 (size=12001) 2024-11-07T15:24:36,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:36,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993136413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993136414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:36,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993136445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:36,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:36,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993136445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993136446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T15:24:36,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:24:36,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:36,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:36,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:36,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:36,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:36,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993136551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993136551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:36,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993136561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993136561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993136561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:36,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:24:36,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:36,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:36,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:36,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:36,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T15:24:36,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993136761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993136761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:36,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993136770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993136770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:36,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993136770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:36,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/77a9bfbf41134415bafaf4dba9c8c99f 2024-11-07T15:24:36,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:24:36,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:36,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:36,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:36,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:36,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:36,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/82be4ffc74bf4b858aa57d8507858178 is 50, key is test_row_0/B:col10/1730993076366/Put/seqid=0 2024-11-07T15:24:36,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742260_1436 (size=12001) 2024-11-07T15:24:36,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:24:36,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:36,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:36,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:36,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:36,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:36,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T15:24:37,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993137071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:37,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:37,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993137071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:37,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993137081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:37,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993137081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:37,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993137081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:37,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:24:37,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:37,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:37,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:37,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:37,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:37,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/82be4ffc74bf4b858aa57d8507858178 2024-11-07T15:24:37,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/6af164766da74fe094cbeb37c741c026 is 50, key is test_row_0/C:col10/1730993076366/Put/seqid=0 2024-11-07T15:24:37,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742261_1437 (size=12001) 2024-11-07T15:24:37,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:24:37,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:37,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:37,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:37,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:37,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:37,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:24:37,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:37,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:37,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:37,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:37,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T15:24:37,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:24:37,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:37,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:37,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:37,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:37,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:37,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:37,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:37,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993137581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993137581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:37,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:37,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993137590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:37,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:37,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993137590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993137590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:37,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/6af164766da74fe094cbeb37c741c026 2024-11-07T15:24:37,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/77a9bfbf41134415bafaf4dba9c8c99f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/77a9bfbf41134415bafaf4dba9c8c99f 2024-11-07T15:24:37,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/77a9bfbf41134415bafaf4dba9c8c99f, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T15:24:37,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/82be4ffc74bf4b858aa57d8507858178 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/82be4ffc74bf4b858aa57d8507858178 2024-11-07T15:24:37,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/82be4ffc74bf4b858aa57d8507858178, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T15:24:37,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/6af164766da74fe094cbeb37c741c026 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/6af164766da74fe094cbeb37c741c026 2024-11-07T15:24:37,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/6af164766da74fe094cbeb37c741c026, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T15:24:37,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for a4f40fe465370ef7d04dfc7daeebbbb9 in 1301ms, sequenceid=15, compaction requested=false 2024-11-07T15:24:37,669 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T15:24:37,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:37,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:24:37,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:37,730 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T15:24:37,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/b050a2ffe51444aa9624d93096448463 is 50, key is test_row_0/A:col10/1730993076430/Put/seqid=0 2024-11-07T15:24:37,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742262_1438 (size=12001) 2024-11-07T15:24:38,146 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/b050a2ffe51444aa9624d93096448463 2024-11-07T15:24:38,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/9565d81f5cbc4a038be929f54800185e is 50, key is test_row_0/B:col10/1730993076430/Put/seqid=0 2024-11-07T15:24:38,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742263_1439 (size=12001) 2024-11-07T15:24:38,156 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/9565d81f5cbc4a038be929f54800185e 2024-11-07T15:24:38,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/23ced06617c649978ed0fb7553cb7730 is 50, key is test_row_0/C:col10/1730993076430/Put/seqid=0 2024-11-07T15:24:38,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742264_1440 (size=12001) 2024-11-07T15:24:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T15:24:38,569 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/23ced06617c649978ed0fb7553cb7730 2024-11-07T15:24:38,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/b050a2ffe51444aa9624d93096448463 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/b050a2ffe51444aa9624d93096448463 2024-11-07T15:24:38,579 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/b050a2ffe51444aa9624d93096448463, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T15:24:38,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/9565d81f5cbc4a038be929f54800185e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/9565d81f5cbc4a038be929f54800185e 2024-11-07T15:24:38,583 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/9565d81f5cbc4a038be929f54800185e, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T15:24:38,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/23ced06617c649978ed0fb7553cb7730 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/23ced06617c649978ed0fb7553cb7730 2024-11-07T15:24:38,586 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/23ced06617c649978ed0fb7553cb7730, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T15:24:38,587 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for a4f40fe465370ef7d04dfc7daeebbbb9 in 857ms, sequenceid=38, compaction requested=false 2024-11-07T15:24:38,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:38,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:38,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-07T15:24:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-07T15:24:38,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-07T15:24:38,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2340 sec 2024-11-07T15:24:38,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.2410 sec 2024-11-07T15:24:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:38,608 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:38,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:24:38,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/e43e78ee97334a758a554e075aa79270 is 50, key is test_row_0/A:col10/1730993078606/Put/seqid=0 2024-11-07T15:24:38,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742265_1441 (size=14341) 2024-11-07T15:24:38,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993138647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:38,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993138654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:38,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993138655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:38,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993138655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993138656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:38,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993138760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993138760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:38,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993138771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:38,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993138771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993138771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:38,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993138970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993138970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:38,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993138981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993138981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:38,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993138981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:39,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/e43e78ee97334a758a554e075aa79270 2024-11-07T15:24:39,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/40d75a7ce5614dfe92834e32ca7be074 is 50, key is test_row_0/B:col10/1730993078606/Put/seqid=0 2024-11-07T15:24:39,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742266_1442 (size=12001) 2024-11-07T15:24:39,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993139281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993139281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:39,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993139290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993139290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:39,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993139290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:39,342 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T15:24:39,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/40d75a7ce5614dfe92834e32ca7be074 2024-11-07T15:24:39,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/2b6c471cc2234612ad38670d62d8d838 is 50, key is test_row_0/C:col10/1730993078606/Put/seqid=0 2024-11-07T15:24:39,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742267_1443 (size=12001) 2024-11-07T15:24:39,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993139791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:39,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:39,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993139800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:39,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993139800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993139800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:39,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993139801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:39,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/2b6c471cc2234612ad38670d62d8d838 2024-11-07T15:24:39,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/e43e78ee97334a758a554e075aa79270 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e43e78ee97334a758a554e075aa79270 2024-11-07T15:24:39,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e43e78ee97334a758a554e075aa79270, entries=200, sequenceid=49, filesize=14.0 K 2024-11-07T15:24:39,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/40d75a7ce5614dfe92834e32ca7be074 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/40d75a7ce5614dfe92834e32ca7be074 2024-11-07T15:24:39,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/40d75a7ce5614dfe92834e32ca7be074, entries=150, sequenceid=49, filesize=11.7 K 2024-11-07T15:24:39,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/2b6c471cc2234612ad38670d62d8d838 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2b6c471cc2234612ad38670d62d8d838 2024-11-07T15:24:39,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2b6c471cc2234612ad38670d62d8d838, entries=150, sequenceid=49, filesize=11.7 K 2024-11-07T15:24:39,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a4f40fe465370ef7d04dfc7daeebbbb9 in 1247ms, sequenceid=49, compaction requested=true 2024-11-07T15:24:39,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:39,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:24:39,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:39,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:24:39,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:39,856 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:39,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:24:39,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:39,856 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:39,856 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:39,856 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:39,857 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/A is initiating minor compaction (all files) 2024-11-07T15:24:39,857 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/B is initiating minor compaction (all files) 2024-11-07T15:24:39,857 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/A in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:39,857 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/B in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:39,857 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/77a9bfbf41134415bafaf4dba9c8c99f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/b050a2ffe51444aa9624d93096448463, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e43e78ee97334a758a554e075aa79270] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=37.4 K 2024-11-07T15:24:39,857 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/82be4ffc74bf4b858aa57d8507858178, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/9565d81f5cbc4a038be929f54800185e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/40d75a7ce5614dfe92834e32ca7be074] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=35.2 K 2024-11-07T15:24:39,857 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77a9bfbf41134415bafaf4dba9c8c99f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730993076366 2024-11-07T15:24:39,857 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 82be4ffc74bf4b858aa57d8507858178, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730993076366 2024-11-07T15:24:39,857 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting b050a2ffe51444aa9624d93096448463, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730993076410 2024-11-07T15:24:39,857 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting e43e78ee97334a758a554e075aa79270, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1730993078602 2024-11-07T15:24:39,858 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 9565d81f5cbc4a038be929f54800185e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730993076410 2024-11-07T15:24:39,858 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 40d75a7ce5614dfe92834e32ca7be074, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1730993078602 2024-11-07T15:24:39,863 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#A#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:39,863 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/fcfabbd3779a4ed4a7bfabc6ad913df4 is 50, key is test_row_0/A:col10/1730993078606/Put/seqid=0 2024-11-07T15:24:39,864 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#B#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:39,864 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/e29390e07db647d4891f7d4e5351015f is 50, key is test_row_0/B:col10/1730993078606/Put/seqid=0 2024-11-07T15:24:39,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742268_1444 (size=12104) 2024-11-07T15:24:39,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742269_1445 (size=12104) 2024-11-07T15:24:39,877 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/fcfabbd3779a4ed4a7bfabc6ad913df4 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/fcfabbd3779a4ed4a7bfabc6ad913df4 2024-11-07T15:24:39,883 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/A of a4f40fe465370ef7d04dfc7daeebbbb9 into fcfabbd3779a4ed4a7bfabc6ad913df4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:39,883 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:39,883 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/A, priority=13, startTime=1730993079855; duration=0sec 2024-11-07T15:24:39,883 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:39,883 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:A 2024-11-07T15:24:39,883 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:39,884 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:39,884 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/C is initiating minor compaction (all files) 2024-11-07T15:24:39,884 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/C in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:39,884 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/6af164766da74fe094cbeb37c741c026, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/23ced06617c649978ed0fb7553cb7730, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2b6c471cc2234612ad38670d62d8d838] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=35.2 K 2024-11-07T15:24:39,884 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6af164766da74fe094cbeb37c741c026, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730993076366 2024-11-07T15:24:39,884 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23ced06617c649978ed0fb7553cb7730, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730993076410 2024-11-07T15:24:39,885 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b6c471cc2234612ad38670d62d8d838, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1730993078602 2024-11-07T15:24:39,892 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#C#compaction#369 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:39,892 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/030fa26683fe4fbfaf17c2050011d060 is 50, key is test_row_0/C:col10/1730993078606/Put/seqid=0 2024-11-07T15:24:39,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742270_1446 (size=12104) 2024-11-07T15:24:40,277 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/e29390e07db647d4891f7d4e5351015f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/e29390e07db647d4891f7d4e5351015f 2024-11-07T15:24:40,281 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/B of a4f40fe465370ef7d04dfc7daeebbbb9 into e29390e07db647d4891f7d4e5351015f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:40,281 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:40,281 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/B, priority=13, startTime=1730993079855; duration=0sec 2024-11-07T15:24:40,281 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:40,281 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:B 2024-11-07T15:24:40,303 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/030fa26683fe4fbfaf17c2050011d060 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/030fa26683fe4fbfaf17c2050011d060 2024-11-07T15:24:40,307 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/C of a4f40fe465370ef7d04dfc7daeebbbb9 into 030fa26683fe4fbfaf17c2050011d060(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:40,307 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:40,307 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/C, priority=13, startTime=1730993079856; duration=0sec 2024-11-07T15:24:40,307 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:40,307 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:C 2024-11-07T15:24:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T15:24:40,521 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:40,521 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-07T15:24:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T15:24:40,523 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:40,523 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:40,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:40,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T15:24:40,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T15:24:40,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:40,676 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T15:24:40,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/df6ffa94f965454f900fd1b0b1cf8d8c is 50, key is test_row_0/A:col10/1730993078654/Put/seqid=0 2024-11-07T15:24:40,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742271_1447 (size=12001) 2024-11-07T15:24:40,684 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/df6ffa94f965454f900fd1b0b1cf8d8c 2024-11-07T15:24:40,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/8fab5e8462da451595bd5e4cdd3be3fe is 50, key is test_row_0/B:col10/1730993078654/Put/seqid=0 2024-11-07T15:24:40,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742272_1448 (size=12001) 2024-11-07T15:24:40,694 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/8fab5e8462da451595bd5e4cdd3be3fe 2024-11-07T15:24:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/1527f1fc55ed4074bf2be58df1ed1088 is 50, key is test_row_0/C:col10/1730993078654/Put/seqid=0 2024-11-07T15:24:40,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742273_1449 (size=12001) 2024-11-07T15:24:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:40,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993140820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:40,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993140823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:40,825 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in a4f40fe465370ef7d04dfc7daeebbbb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:40,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993140824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:40,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:40,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993140825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:40,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:40,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993140825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:40,833 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,833 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,833 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T15:24:40,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993140930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:40,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993140940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:40,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993140941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:40,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993140941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:41,105 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/1527f1fc55ed4074bf2be58df1ed1088 2024-11-07T15:24:41,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/df6ffa94f965454f900fd1b0b1cf8d8c as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/df6ffa94f965454f900fd1b0b1cf8d8c 2024-11-07T15:24:41,111 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/df6ffa94f965454f900fd1b0b1cf8d8c, entries=150, sequenceid=77, filesize=11.7 K 2024-11-07T15:24:41,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/8fab5e8462da451595bd5e4cdd3be3fe as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/8fab5e8462da451595bd5e4cdd3be3fe 2024-11-07T15:24:41,115 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/8fab5e8462da451595bd5e4cdd3be3fe, entries=150, sequenceid=77, filesize=11.7 K 2024-11-07T15:24:41,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/1527f1fc55ed4074bf2be58df1ed1088 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/1527f1fc55ed4074bf2be58df1ed1088 2024-11-07T15:24:41,120 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/1527f1fc55ed4074bf2be58df1ed1088, entries=150, sequenceid=77, filesize=11.7 K 2024-11-07T15:24:41,121 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a4f40fe465370ef7d04dfc7daeebbbb9 in 445ms, sequenceid=77, compaction requested=false 2024-11-07T15:24:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-07T15:24:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-07T15:24:41,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-07T15:24:41,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 600 msec 2024-11-07T15:24:41,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 606 msec 2024-11-07T15:24:41,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:41,143 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:41,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T15:24:41,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/e125febe38014b4fa403efa82a104729 is 50, key is test_row_0/A:col10/1730993080823/Put/seqid=0 2024-11-07T15:24:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T15:24:41,151 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:41,152 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-07T15:24:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T15:24:41,155 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:41,155 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:41,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:41,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742274_1450 (size=11997) 2024-11-07T15:24:41,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/e125febe38014b4fa403efa82a104729 2024-11-07T15:24:41,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/f0cbc1c656274c91a505b256081f7078 is 50, key is test_row_0/B:col10/1730993080823/Put/seqid=0 2024-11-07T15:24:41,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742275_1451 (size=9657) 2024-11-07T15:24:41,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/f0cbc1c656274c91a505b256081f7078 2024-11-07T15:24:41,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/8d91262b1f2f491094fb0e17960cb0c3 is 50, key is test_row_0/C:col10/1730993080823/Put/seqid=0 2024-11-07T15:24:41,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742276_1452 (size=9657) 2024-11-07T15:24:41,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993141198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993141198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:41,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993141199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993141200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:41,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T15:24:41,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-07T15:24:41,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:41,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:41,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:41,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:41,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:41,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993141311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:41,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:41,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993141311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:41,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:41,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993141311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:41,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993141312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:41,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-07T15:24:41,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:41,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:41,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:41,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:41,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T15:24:41,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993141520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:41,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:41,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993141521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:41,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993141531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:41,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993141531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:41,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:41,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:41,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/8d91262b1f2f491094fb0e17960cb0c3 2024-11-07T15:24:41,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/e125febe38014b4fa403efa82a104729 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e125febe38014b4fa403efa82a104729 2024-11-07T15:24:41,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e125febe38014b4fa403efa82a104729, entries=150, sequenceid=89, filesize=11.7 K 2024-11-07T15:24:41,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/f0cbc1c656274c91a505b256081f7078 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/f0cbc1c656274c91a505b256081f7078 2024-11-07T15:24:41,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-07T15:24:41,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/f0cbc1c656274c91a505b256081f7078, entries=100, sequenceid=89, filesize=9.4 K 2024-11-07T15:24:41,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:41,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:41,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:41,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:41,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:41,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/8d91262b1f2f491094fb0e17960cb0c3 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/8d91262b1f2f491094fb0e17960cb0c3 2024-11-07T15:24:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:41,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/8d91262b1f2f491094fb0e17960cb0c3, entries=100, sequenceid=89, filesize=9.4 K 2024-11-07T15:24:41,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a4f40fe465370ef7d04dfc7daeebbbb9 in 493ms, sequenceid=89, compaction requested=true 2024-11-07T15:24:41,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:41,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:24:41,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:41,637 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:41,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:24:41,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:41,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:24:41,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T15:24:41,637 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:41,641 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36102 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:41,641 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/A is initiating minor compaction (all files) 2024-11-07T15:24:41,641 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/A in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:41,641 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/fcfabbd3779a4ed4a7bfabc6ad913df4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/df6ffa94f965454f900fd1b0b1cf8d8c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e125febe38014b4fa403efa82a104729] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=35.3 K 2024-11-07T15:24:41,641 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:41,641 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/B is initiating minor compaction (all files) 2024-11-07T15:24:41,642 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/B in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:41,642 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/e29390e07db647d4891f7d4e5351015f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/8fab5e8462da451595bd5e4cdd3be3fe, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/f0cbc1c656274c91a505b256081f7078] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=33.0 K 2024-11-07T15:24:41,642 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcfabbd3779a4ed4a7bfabc6ad913df4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1730993078602 2024-11-07T15:24:41,642 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting e29390e07db647d4891f7d4e5351015f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1730993078602 2024-11-07T15:24:41,642 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fab5e8462da451595bd5e4cdd3be3fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730993078652 2024-11-07T15:24:41,642 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting df6ffa94f965454f900fd1b0b1cf8d8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730993078652 2024-11-07T15:24:41,643 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting f0cbc1c656274c91a505b256081f7078, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1730993080823 2024-11-07T15:24:41,645 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting e125febe38014b4fa403efa82a104729, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1730993080823 2024-11-07T15:24:41,658 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#B#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:41,658 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/84403c51bd1f479592a6d2df659891f0 is 50, key is test_row_0/B:col10/1730993080823/Put/seqid=0 2024-11-07T15:24:41,670 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#A#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:41,671 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/9f4a658f0ab64571a6ee026a6b1d32b5 is 50, key is test_row_0/A:col10/1730993080823/Put/seqid=0 2024-11-07T15:24:41,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742277_1453 (size=12207) 2024-11-07T15:24:41,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742278_1454 (size=12207) 2024-11-07T15:24:41,681 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/84403c51bd1f479592a6d2df659891f0 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/84403c51bd1f479592a6d2df659891f0 2024-11-07T15:24:41,684 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/9f4a658f0ab64571a6ee026a6b1d32b5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/9f4a658f0ab64571a6ee026a6b1d32b5 2024-11-07T15:24:41,687 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/B of a4f40fe465370ef7d04dfc7daeebbbb9 into 84403c51bd1f479592a6d2df659891f0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:41,687 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:41,687 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/B, priority=13, startTime=1730993081637; duration=0sec 2024-11-07T15:24:41,688 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:41,688 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:B 2024-11-07T15:24:41,688 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:41,693 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/A of a4f40fe465370ef7d04dfc7daeebbbb9 into 9f4a658f0ab64571a6ee026a6b1d32b5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:41,693 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:41,693 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/A, priority=13, startTime=1730993081637; duration=0sec 2024-11-07T15:24:41,693 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:41,693 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:A 2024-11-07T15:24:41,702 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:41,702 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/C is initiating minor compaction (all files) 2024-11-07T15:24:41,702 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/C in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:41,702 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/030fa26683fe4fbfaf17c2050011d060, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/1527f1fc55ed4074bf2be58df1ed1088, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/8d91262b1f2f491094fb0e17960cb0c3] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=33.0 K 2024-11-07T15:24:41,702 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 030fa26683fe4fbfaf17c2050011d060, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1730993078602 2024-11-07T15:24:41,703 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 1527f1fc55ed4074bf2be58df1ed1088, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730993078652 2024-11-07T15:24:41,703 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d91262b1f2f491094fb0e17960cb0c3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1730993080823 2024-11-07T15:24:41,709 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#C#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:41,710 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/2fbde78f8a024687b98602bb6d05acf2 is 50, key is test_row_0/C:col10/1730993080823/Put/seqid=0 2024-11-07T15:24:41,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742279_1455 (size=12207) 2024-11-07T15:24:41,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-07T15:24:41,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:41,769 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T15:24:41,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/f86f0f3868e240c5996fbb9aa599a9f3 is 50, key is test_row_0/A:col10/1730993081199/Put/seqid=0 2024-11-07T15:24:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T15:24:41,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742280_1456 (size=12001) 2024-11-07T15:24:41,800 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/f86f0f3868e240c5996fbb9aa599a9f3 2024-11-07T15:24:41,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/72de5b845fbd4e5490e4b1703ef6206a is 50, key is test_row_0/B:col10/1730993081199/Put/seqid=0 2024-11-07T15:24:41,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742281_1457 (size=12001) 2024-11-07T15:24:41,837 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/72de5b845fbd4e5490e4b1703ef6206a 2024-11-07T15:24:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:41,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:41,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/dd9abfdb37134957aeb1aad8ff9aadc4 is 50, key is test_row_0/C:col10/1730993081199/Put/seqid=0 2024-11-07T15:24:41,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742282_1458 (size=12001) 2024-11-07T15:24:41,858 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/dd9abfdb37134957aeb1aad8ff9aadc4 2024-11-07T15:24:41,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/f86f0f3868e240c5996fbb9aa599a9f3 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/f86f0f3868e240c5996fbb9aa599a9f3 2024-11-07T15:24:41,866 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/f86f0f3868e240c5996fbb9aa599a9f3, entries=150, sequenceid=115, filesize=11.7 K 2024-11-07T15:24:41,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/72de5b845fbd4e5490e4b1703ef6206a as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/72de5b845fbd4e5490e4b1703ef6206a 2024-11-07T15:24:41,873 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/72de5b845fbd4e5490e4b1703ef6206a, entries=150, sequenceid=115, filesize=11.7 K 2024-11-07T15:24:41,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/dd9abfdb37134957aeb1aad8ff9aadc4 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/dd9abfdb37134957aeb1aad8ff9aadc4 2024-11-07T15:24:41,880 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/dd9abfdb37134957aeb1aad8ff9aadc4, entries=150, sequenceid=115, filesize=11.7 K 2024-11-07T15:24:41,886 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=53.67 KB/54960 for a4f40fe465370ef7d04dfc7daeebbbb9 in 117ms, sequenceid=115, compaction requested=false 2024-11-07T15:24:41,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:41,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:41,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-07T15:24:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-07T15:24:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:41,888 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:41,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T15:24:41,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-07T15:24:41,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 733 msec 2024-11-07T15:24:41,896 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 742 msec 2024-11-07T15:24:41,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/7fa64416106e44bd9b17d7396dacf01b is 50, key is test_row_0/A:col10/1730993081879/Put/seqid=0 2024-11-07T15:24:41,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742283_1459 (size=12001) 2024-11-07T15:24:41,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993141964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:41,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:41,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993141966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:41,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:41,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:41,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993141967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:41,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993141978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:42,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993142082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:42,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993142090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993142091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:42,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993142101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:42,134 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/2fbde78f8a024687b98602bb6d05acf2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2fbde78f8a024687b98602bb6d05acf2 2024-11-07T15:24:42,143 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/C of a4f40fe465370ef7d04dfc7daeebbbb9 into 2fbde78f8a024687b98602bb6d05acf2(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:42,143 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:42,143 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/C, priority=13, startTime=1730993081637; duration=0sec 2024-11-07T15:24:42,143 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:42,143 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:C 2024-11-07T15:24:42,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T15:24:42,291 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:42,292 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:42,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-07T15:24:42,293 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:42,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T15:24:42,294 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:42,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:42,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993142301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:42,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993142301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:42,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:42,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993142301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:42,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/7fa64416106e44bd9b17d7396dacf01b 2024-11-07T15:24:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:42,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/ac0807dd4de042b1a9cbecd38d1a91e1 is 50, key is test_row_0/B:col10/1730993081879/Put/seqid=0 2024-11-07T15:24:42,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742284_1460 (size=12001) 2024-11-07T15:24:42,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993142321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T15:24:42,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T15:24:42,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:42,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:42,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:42,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:42,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T15:24:42,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:42,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:42,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:42,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:42,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T15:24:42,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993142611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:42,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993142611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993142611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:42,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993142641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:42,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/ac0807dd4de042b1a9cbecd38d1a91e1 2024-11-07T15:24:42,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/771866541f8744a9b2f7728d08625316 is 50, key is test_row_0/C:col10/1730993081879/Put/seqid=0 2024-11-07T15:24:42,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742285_1461 (size=12001) 2024-11-07T15:24:42,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T15:24:42,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/771866541f8744a9b2f7728d08625316 2024-11-07T15:24:42,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:42,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:42,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:42,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:42,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:42,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/7fa64416106e44bd9b17d7396dacf01b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/7fa64416106e44bd9b17d7396dacf01b 2024-11-07T15:24:42,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/7fa64416106e44bd9b17d7396dacf01b, entries=150, sequenceid=128, filesize=11.7 K 2024-11-07T15:24:42,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/ac0807dd4de042b1a9cbecd38d1a91e1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/ac0807dd4de042b1a9cbecd38d1a91e1 2024-11-07T15:24:42,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/ac0807dd4de042b1a9cbecd38d1a91e1, entries=150, sequenceid=128, filesize=11.7 K 2024-11-07T15:24:42,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/771866541f8744a9b2f7728d08625316 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/771866541f8744a9b2f7728d08625316 2024-11-07T15:24:42,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/771866541f8744a9b2f7728d08625316, entries=150, sequenceid=128, filesize=11.7 K 2024-11-07T15:24:42,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a4f40fe465370ef7d04dfc7daeebbbb9 in 885ms, sequenceid=128, compaction requested=true 2024-11-07T15:24:42,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:42,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:24:42,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:42,773 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:42,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:24:42,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:42,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:24:42,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T15:24:42,773 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:42,776 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:42,776 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/B is initiating minor compaction (all files) 2024-11-07T15:24:42,776 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:42,776 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/A is initiating minor compaction (all files) 2024-11-07T15:24:42,776 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/B in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:42,776 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/A in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:42,776 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/84403c51bd1f479592a6d2df659891f0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/72de5b845fbd4e5490e4b1703ef6206a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/ac0807dd4de042b1a9cbecd38d1a91e1] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=35.4 K 2024-11-07T15:24:42,776 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/9f4a658f0ab64571a6ee026a6b1d32b5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/f86f0f3868e240c5996fbb9aa599a9f3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/7fa64416106e44bd9b17d7396dacf01b] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=35.4 K 2024-11-07T15:24:42,777 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 84403c51bd1f479592a6d2df659891f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1730993078652 2024-11-07T15:24:42,777 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f4a658f0ab64571a6ee026a6b1d32b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1730993078652 2024-11-07T15:24:42,777 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 72de5b845fbd4e5490e4b1703ef6206a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730993081171 2024-11-07T15:24:42,777 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting f86f0f3868e240c5996fbb9aa599a9f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730993081171 2024-11-07T15:24:42,777 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting ac0807dd4de042b1a9cbecd38d1a91e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1730993081867 2024-11-07T15:24:42,777 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fa64416106e44bd9b17d7396dacf01b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1730993081867 2024-11-07T15:24:42,786 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#A#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:42,787 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/b9aed944c39841aeabec4c55696e2704 is 50, key is test_row_0/A:col10/1730993081879/Put/seqid=0 2024-11-07T15:24:42,787 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#B#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:42,788 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/7069b470a7f34799a4f311a8ae0326be is 50, key is test_row_0/B:col10/1730993081879/Put/seqid=0 2024-11-07T15:24:42,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742286_1462 (size=12309) 2024-11-07T15:24:42,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742287_1463 (size=12309) 2024-11-07T15:24:42,811 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/7069b470a7f34799a4f311a8ae0326be as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/7069b470a7f34799a4f311a8ae0326be 2024-11-07T15:24:42,817 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/B of a4f40fe465370ef7d04dfc7daeebbbb9 into 7069b470a7f34799a4f311a8ae0326be(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:42,817 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:42,817 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/B, priority=13, startTime=1730993082773; duration=0sec 2024-11-07T15:24:42,817 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:42,817 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:B 2024-11-07T15:24:42,817 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:42,818 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:42,818 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/C is initiating minor compaction (all files) 2024-11-07T15:24:42,818 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/C in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:42,818 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2fbde78f8a024687b98602bb6d05acf2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/dd9abfdb37134957aeb1aad8ff9aadc4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/771866541f8744a9b2f7728d08625316] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=35.4 K 2024-11-07T15:24:42,819 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fbde78f8a024687b98602bb6d05acf2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1730993078652 2024-11-07T15:24:42,819 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting dd9abfdb37134957aeb1aad8ff9aadc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1730993081171 2024-11-07T15:24:42,819 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 771866541f8744a9b2f7728d08625316, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1730993081867 2024-11-07T15:24:42,825 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#C#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:42,825 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/123912095ecc467397366e6c5d28eaf3 is 50, key is test_row_0/C:col10/1730993081879/Put/seqid=0 2024-11-07T15:24:42,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742288_1464 (size=12309) 2024-11-07T15:24:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:42,845 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:42,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T15:24:42,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/4cea5b1924274a779b624552b2f62588 is 50, key is test_row_0/A:col10/1730993081975/Put/seqid=0 2024-11-07T15:24:42,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742289_1465 (size=14541) 2024-11-07T15:24:42,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T15:24:42,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:42,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:42,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:42,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:42,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T15:24:42,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993142923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:43,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993143031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:43,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:43,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:43,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:43,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T15:24:43,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:43,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:43,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:43,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993143121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:43,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993143121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:43,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993143121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:43,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:43,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993143160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:43,203 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/b9aed944c39841aeabec4c55696e2704 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/b9aed944c39841aeabec4c55696e2704 2024-11-07T15:24:43,207 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/A of a4f40fe465370ef7d04dfc7daeebbbb9 into b9aed944c39841aeabec4c55696e2704(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:43,207 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:43,207 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/A, priority=13, startTime=1730993082773; duration=0sec 2024-11-07T15:24:43,207 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:43,207 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:A 2024-11-07T15:24:43,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T15:24:43,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:43,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:43,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:43,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,238 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/123912095ecc467397366e6c5d28eaf3 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/123912095ecc467397366e6c5d28eaf3 2024-11-07T15:24:43,242 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/C of a4f40fe465370ef7d04dfc7daeebbbb9 into 123912095ecc467397366e6c5d28eaf3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:43,242 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:43,242 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/C, priority=13, startTime=1730993082773; duration=0sec 2024-11-07T15:24:43,242 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:43,242 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:C 2024-11-07T15:24:43,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993143240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:43,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:43,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:43,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:43,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/4cea5b1924274a779b624552b2f62588 2024-11-07T15:24:43,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/47259e67e4be4c29b6ecfbaaa23a23c2 is 50, key is test_row_0/B:col10/1730993081975/Put/seqid=0 2024-11-07T15:24:43,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742290_1466 (size=12151) 2024-11-07T15:24:43,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T15:24:43,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:43,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:43,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:43,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T15:24:43,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T15:24:43,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:43,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:43,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:43,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993143550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:43,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T15:24:43,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:43,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:43,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:43,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/47259e67e4be4c29b6ecfbaaa23a23c2 2024-11-07T15:24:43,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/a504c00140b640c4833fae38df977bfb is 50, key is test_row_0/C:col10/1730993081975/Put/seqid=0 2024-11-07T15:24:43,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742291_1467 (size=12151) 2024-11-07T15:24:43,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T15:24:43,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:43,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:43,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:43,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T15:24:43,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:43,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:43,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:43,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:44,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993144060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/a504c00140b640c4833fae38df977bfb 2024-11-07T15:24:44,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/4cea5b1924274a779b624552b2f62588 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/4cea5b1924274a779b624552b2f62588 2024-11-07T15:24:44,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/4cea5b1924274a779b624552b2f62588, entries=200, sequenceid=154, filesize=14.2 K 2024-11-07T15:24:44,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/47259e67e4be4c29b6ecfbaaa23a23c2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/47259e67e4be4c29b6ecfbaaa23a23c2 2024-11-07T15:24:44,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/47259e67e4be4c29b6ecfbaaa23a23c2, entries=150, sequenceid=154, filesize=11.9 K 2024-11-07T15:24:44,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/a504c00140b640c4833fae38df977bfb as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/a504c00140b640c4833fae38df977bfb 2024-11-07T15:24:44,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/a504c00140b640c4833fae38df977bfb, entries=150, sequenceid=154, filesize=11.9 K 2024-11-07T15:24:44,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for a4f40fe465370ef7d04dfc7daeebbbb9 in 1254ms, sequenceid=154, compaction requested=false 2024-11-07T15:24:44,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:44,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T15:24:44,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:44,127 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T15:24:44,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/c15a9a8ff5ed48159cf3a2daab4553f6 is 50, key is test_row_0/A:col10/1730993082877/Put/seqid=0 2024-11-07T15:24:44,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742292_1468 (size=12151) 2024-11-07T15:24:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:44,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:44,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993144190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993144191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993144198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993144199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993144310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993144311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993144311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993144311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T15:24:44,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993144520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993144521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993144521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993144521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,536 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/c15a9a8ff5ed48159cf3a2daab4553f6 2024-11-07T15:24:44,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/6d37c09790b54409b825b47892d45b30 is 50, key is test_row_0/B:col10/1730993082877/Put/seqid=0 2024-11-07T15:24:44,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742293_1469 (size=12151) 2024-11-07T15:24:44,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993144831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993144831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993144831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993144831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:44,945 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/6d37c09790b54409b825b47892d45b30 2024-11-07T15:24:44,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/2803fcac15da4f2c86c2f2b8dcdd80ec is 50, key is test_row_0/C:col10/1730993082877/Put/seqid=0 2024-11-07T15:24:44,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742294_1470 (size=12151) 2024-11-07T15:24:45,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:45,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993145070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:45,076 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_0' in a4f40fe465370ef7d04dfc7daeebbbb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:45,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:45,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:45,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:45,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993145340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:45,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993145340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:45,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:45,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:45,353 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/2803fcac15da4f2c86c2f2b8dcdd80ec 2024-11-07T15:24:45,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/c15a9a8ff5ed48159cf3a2daab4553f6 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/c15a9a8ff5ed48159cf3a2daab4553f6 2024-11-07T15:24:45,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993145350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:45,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:45,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:45,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:45,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993145351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:45,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:45,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:45,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:45,360 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/c15a9a8ff5ed48159cf3a2daab4553f6, entries=150, sequenceid=168, filesize=11.9 K 2024-11-07T15:24:45,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/6d37c09790b54409b825b47892d45b30 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/6d37c09790b54409b825b47892d45b30 2024-11-07T15:24:45,363 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/6d37c09790b54409b825b47892d45b30, entries=150, sequenceid=168, filesize=11.9 K 2024-11-07T15:24:45,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/2803fcac15da4f2c86c2f2b8dcdd80ec as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2803fcac15da4f2c86c2f2b8dcdd80ec 2024-11-07T15:24:45,366 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2803fcac15da4f2c86c2f2b8dcdd80ec, entries=150, sequenceid=168, filesize=11.9 K 2024-11-07T15:24:45,367 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a4f40fe465370ef7d04dfc7daeebbbb9 in 1240ms, sequenceid=168, compaction requested=true 2024-11-07T15:24:45,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:45,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:45,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-07T15:24:45,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-07T15:24:45,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-07T15:24:45,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0740 sec 2024-11-07T15:24:45,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 3.0780 sec 2024-11-07T15:24:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:46,358 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:46,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T15:24:46,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/25d394eaf82d443793659e60ca8247f6 is 50, key is test_row_0/A:col10/1730993084198/Put/seqid=0 2024-11-07T15:24:46,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742295_1471 (size=14541) 2024-11-07T15:24:46,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993146377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993146383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:46,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993146383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:46,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993146384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:46,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T15:24:46,461 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:46,462 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-07T15:24:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T15:24:46,463 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:46,463 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:46,463 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:46,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993146490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:46,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:46,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993146500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:46,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:46,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993146500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:46,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:46,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993146501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T15:24:46,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T15:24:46,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:46,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:46,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:46,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:46,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:46,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993146700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:46,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993146710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993146711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:46,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993146711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:46,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T15:24:46,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/25d394eaf82d443793659e60ca8247f6 2024-11-07T15:24:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:46,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:46,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/238b888fb8c7443f815d94384d6667a8 is 50, key is test_row_0/B:col10/1730993084198/Put/seqid=0 2024-11-07T15:24:46,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742296_1472 (size=12151) 2024-11-07T15:24:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T15:24:46,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T15:24:46,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:46,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:46,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:46,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:46,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:47,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:47,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993147010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:47,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:47,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993147020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993147020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:47,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:47,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993147031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:47,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:47,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T15:24:47,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:47,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:47,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:47,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:47,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:47,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993147080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:47,084 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_0' in a4f40fe465370ef7d04dfc7daeebbbb9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T15:24:47,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/238b888fb8c7443f815d94384d6667a8 2024-11-07T15:24:47,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/49cffa281d094c75be214486f89ef174 is 50, key is test_row_0/C:col10/1730993084198/Put/seqid=0 2024-11-07T15:24:47,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742297_1473 (size=12151) 2024-11-07T15:24:47,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T15:24:47,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:47,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:47,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:47,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:47,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:47,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T15:24:47,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:47,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:47,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:47,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:47,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:47,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993147520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:47,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T15:24:47,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:47,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:47,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:47,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:47,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:47,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993147541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:47,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:47,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993147541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:47,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:47,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993147551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:47,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/49cffa281d094c75be214486f89ef174 2024-11-07T15:24:47,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/25d394eaf82d443793659e60ca8247f6 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/25d394eaf82d443793659e60ca8247f6 2024-11-07T15:24:47,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/25d394eaf82d443793659e60ca8247f6, entries=200, sequenceid=193, filesize=14.2 K 2024-11-07T15:24:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/238b888fb8c7443f815d94384d6667a8 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/238b888fb8c7443f815d94384d6667a8 2024-11-07T15:24:47,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/238b888fb8c7443f815d94384d6667a8, entries=150, sequenceid=193, filesize=11.9 K 2024-11-07T15:24:47,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/49cffa281d094c75be214486f89ef174 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/49cffa281d094c75be214486f89ef174 2024-11-07T15:24:47,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/49cffa281d094c75be214486f89ef174, entries=150, sequenceid=193, filesize=11.9 K 2024-11-07T15:24:47,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a4f40fe465370ef7d04dfc7daeebbbb9 in 1241ms, sequenceid=193, compaction requested=true 2024-11-07T15:24:47,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:47,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:24:47,599 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:24:47,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:47,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:24:47,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:47,599 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:24:47,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:24:47,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:47,600 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53542 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:24:47,600 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/A is initiating minor compaction (all files) 2024-11-07T15:24:47,600 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/A in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:47,600 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:24:47,600 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/b9aed944c39841aeabec4c55696e2704, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/4cea5b1924274a779b624552b2f62588, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/c15a9a8ff5ed48159cf3a2daab4553f6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/25d394eaf82d443793659e60ca8247f6] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=52.3 K 2024-11-07T15:24:47,600 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/B is initiating minor compaction (all files) 2024-11-07T15:24:47,600 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/B in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:47,600 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/7069b470a7f34799a4f311a8ae0326be, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/47259e67e4be4c29b6ecfbaaa23a23c2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/6d37c09790b54409b825b47892d45b30, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/238b888fb8c7443f815d94384d6667a8] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=47.6 K 2024-11-07T15:24:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T15:24:47,601 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 7069b470a7f34799a4f311a8ae0326be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1730993081867 2024-11-07T15:24:47,601 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9aed944c39841aeabec4c55696e2704, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1730993081867 2024-11-07T15:24:47,601 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 47259e67e4be4c29b6ecfbaaa23a23c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1730993081963 2024-11-07T15:24:47,601 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cea5b1924274a779b624552b2f62588, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1730993081961 2024-11-07T15:24:47,601 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d37c09790b54409b825b47892d45b30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730993082871 2024-11-07T15:24:47,601 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting c15a9a8ff5ed48159cf3a2daab4553f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730993082871 2024-11-07T15:24:47,601 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 238b888fb8c7443f815d94384d6667a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1730993084183 2024-11-07T15:24:47,601 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25d394eaf82d443793659e60ca8247f6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1730993084183 2024-11-07T15:24:47,607 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#A#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:47,607 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#B#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:47,607 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/85b1318de88d4157a99d6b1283d1dc34 is 50, key is test_row_0/A:col10/1730993084198/Put/seqid=0 2024-11-07T15:24:47,607 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/bd379d613c204a8a84413875340ffc92 is 50, key is test_row_0/B:col10/1730993084198/Put/seqid=0 2024-11-07T15:24:47,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742299_1475 (size=12595) 2024-11-07T15:24:47,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742298_1474 (size=12595) 2024-11-07T15:24:47,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T15:24:47,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:47,684 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:24:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/dc5f789b976c48cca127ddc592391b67 is 50, key is test_row_0/A:col10/1730993086369/Put/seqid=0 2024-11-07T15:24:47,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742300_1476 (size=12151) 2024-11-07T15:24:48,014 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/85b1318de88d4157a99d6b1283d1dc34 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/85b1318de88d4157a99d6b1283d1dc34 2024-11-07T15:24:48,014 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/bd379d613c204a8a84413875340ffc92 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/bd379d613c204a8a84413875340ffc92 2024-11-07T15:24:48,018 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/B of a4f40fe465370ef7d04dfc7daeebbbb9 into bd379d613c204a8a84413875340ffc92(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:48,018 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/A of a4f40fe465370ef7d04dfc7daeebbbb9 into 85b1318de88d4157a99d6b1283d1dc34(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:48,018 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:48,018 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:48,018 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/B, priority=12, startTime=1730993087599; duration=0sec 2024-11-07T15:24:48,018 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/A, priority=12, startTime=1730993087599; duration=0sec 2024-11-07T15:24:48,018 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:48,018 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:48,018 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:A 2024-11-07T15:24:48,018 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:B 2024-11-07T15:24:48,018 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:24:48,023 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:24:48,024 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/C is initiating minor compaction (all files) 2024-11-07T15:24:48,024 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/C in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:48,024 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/123912095ecc467397366e6c5d28eaf3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/a504c00140b640c4833fae38df977bfb, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2803fcac15da4f2c86c2f2b8dcdd80ec, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/49cffa281d094c75be214486f89ef174] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=47.6 K 2024-11-07T15:24:48,024 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 123912095ecc467397366e6c5d28eaf3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1730993081867 2024-11-07T15:24:48,024 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting a504c00140b640c4833fae38df977bfb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1730993081963 2024-11-07T15:24:48,025 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2803fcac15da4f2c86c2f2b8dcdd80ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730993082871 2024-11-07T15:24:48,025 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49cffa281d094c75be214486f89ef174, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1730993084183 2024-11-07T15:24:48,031 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#C#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:48,032 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/b2bd86e35fa54e34881ba3d22b39b67b is 50, key is test_row_0/C:col10/1730993084198/Put/seqid=0 2024-11-07T15:24:48,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742301_1477 (size=12595) 2024-11-07T15:24:48,043 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/b2bd86e35fa54e34881ba3d22b39b67b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/b2bd86e35fa54e34881ba3d22b39b67b 2024-11-07T15:24:48,049 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/C of a4f40fe465370ef7d04dfc7daeebbbb9 into b2bd86e35fa54e34881ba3d22b39b67b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:48,049 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:48,049 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/C, priority=12, startTime=1730993087599; duration=0sec 2024-11-07T15:24:48,049 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:48,049 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:C 2024-11-07T15:24:48,091 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/dc5f789b976c48cca127ddc592391b67 2024-11-07T15:24:48,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/9340c18b3d414225a1ce68762d96cad1 is 50, key is test_row_0/B:col10/1730993086369/Put/seqid=0 2024-11-07T15:24:48,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742302_1478 (size=12151) 2024-11-07T15:24:48,499 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/9340c18b3d414225a1ce68762d96cad1 2024-11-07T15:24:48,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/c0d5cbbfa52a4f9c9507a5b73e30eee5 is 50, key is test_row_0/C:col10/1730993086369/Put/seqid=0 2024-11-07T15:24:48,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742303_1479 (size=12151) 2024-11-07T15:24:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:48,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:48,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993148589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:48,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993148599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:48,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993148601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:48,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993148601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T15:24:48,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993148711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:48,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993148711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:48,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993148711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993148711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:48,908 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/c0d5cbbfa52a4f9c9507a5b73e30eee5 2024-11-07T15:24:48,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993148921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993148921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993148921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:48,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993148921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:49,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/dc5f789b976c48cca127ddc592391b67 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/dc5f789b976c48cca127ddc592391b67 2024-11-07T15:24:49,056 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/dc5f789b976c48cca127ddc592391b67, entries=150, sequenceid=204, filesize=11.9 K 2024-11-07T15:24:49,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/9340c18b3d414225a1ce68762d96cad1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/9340c18b3d414225a1ce68762d96cad1 2024-11-07T15:24:49,060 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/9340c18b3d414225a1ce68762d96cad1, entries=150, sequenceid=204, filesize=11.9 K 2024-11-07T15:24:49,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/c0d5cbbfa52a4f9c9507a5b73e30eee5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/c0d5cbbfa52a4f9c9507a5b73e30eee5 2024-11-07T15:24:49,063 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/c0d5cbbfa52a4f9c9507a5b73e30eee5, entries=150, sequenceid=204, filesize=11.9 K 2024-11-07T15:24:49,064 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for a4f40fe465370ef7d04dfc7daeebbbb9 in 1380ms, sequenceid=204, compaction requested=false 2024-11-07T15:24:49,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:49,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:49,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-07T15:24:49,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-07T15:24:49,067 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-07T15:24:49,067 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6020 sec 2024-11-07T15:24:49,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.6050 sec 2024-11-07T15:24:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:49,235 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:49,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T15:24:49,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/f1e8d4f1050d48ac94c126a96cbdfeab is 50, key is test_row_0/A:col10/1730993088588/Put/seqid=0 2024-11-07T15:24:49,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742304_1480 (size=16931) 2024-11-07T15:24:49,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993149238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993149241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:49,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993149246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:49,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:49,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993149247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:49,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:49,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993149351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:49,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993149351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:49,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:49,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993149361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:49,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:49,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993149361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:49,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:49,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:49,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993149561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:49,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:49,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993149561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:49,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:49,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:49,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993149571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:49,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:49,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993149571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:49,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:49,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/f1e8d4f1050d48ac94c126a96cbdfeab 2024-11-07T15:24:49,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/3ac2f402382c47ce99b5b3fbc685d8cd is 50, key is test_row_0/B:col10/1730993088588/Put/seqid=0 2024-11-07T15:24:49,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742305_1481 (size=12151) 2024-11-07T15:24:49,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993149870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:49,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993149870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:49,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993149881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:49,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993149881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:50,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/3ac2f402382c47ce99b5b3fbc685d8cd 2024-11-07T15:24:50,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/6698168d415245598448f2eeae960a6d is 50, key is test_row_0/C:col10/1730993088588/Put/seqid=0 2024-11-07T15:24:50,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742306_1482 (size=12151) 2024-11-07T15:24:50,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:50,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993150390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:50,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:50,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:50,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993150390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:50,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993150391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:50,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:50,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993150391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:50,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:50,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/6698168d415245598448f2eeae960a6d 2024-11-07T15:24:50,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/f1e8d4f1050d48ac94c126a96cbdfeab as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/f1e8d4f1050d48ac94c126a96cbdfeab 2024-11-07T15:24:50,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/f1e8d4f1050d48ac94c126a96cbdfeab, entries=250, sequenceid=235, filesize=16.5 K 2024-11-07T15:24:50,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/3ac2f402382c47ce99b5b3fbc685d8cd as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/3ac2f402382c47ce99b5b3fbc685d8cd 2024-11-07T15:24:50,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/3ac2f402382c47ce99b5b3fbc685d8cd, entries=150, sequenceid=235, filesize=11.9 K 2024-11-07T15:24:50,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/6698168d415245598448f2eeae960a6d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/6698168d415245598448f2eeae960a6d 2024-11-07T15:24:50,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/6698168d415245598448f2eeae960a6d, entries=150, sequenceid=235, filesize=11.9 K 2024-11-07T15:24:50,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for a4f40fe465370ef7d04dfc7daeebbbb9 in 1242ms, sequenceid=235, compaction requested=true 2024-11-07T15:24:50,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:50,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:24:50,478 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:50,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:50,478 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:50,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:24:50,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:50,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:24:50,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:50,479 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:50,479 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:50,479 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/A is initiating minor compaction (all files) 2024-11-07T15:24:50,479 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/B is initiating minor compaction (all files) 2024-11-07T15:24:50,479 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/A in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:50,479 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/B in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:50,479 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/85b1318de88d4157a99d6b1283d1dc34, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/dc5f789b976c48cca127ddc592391b67, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/f1e8d4f1050d48ac94c126a96cbdfeab] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=40.7 K 2024-11-07T15:24:50,479 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/bd379d613c204a8a84413875340ffc92, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/9340c18b3d414225a1ce68762d96cad1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/3ac2f402382c47ce99b5b3fbc685d8cd] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=36.0 K 2024-11-07T15:24:50,479 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting bd379d613c204a8a84413875340ffc92, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1730993084183 2024-11-07T15:24:50,479 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85b1318de88d4157a99d6b1283d1dc34, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1730993084183 2024-11-07T15:24:50,480 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc5f789b976c48cca127ddc592391b67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730993086369 2024-11-07T15:24:50,480 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 9340c18b3d414225a1ce68762d96cad1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730993086369 2024-11-07T15:24:50,480 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1e8d4f1050d48ac94c126a96cbdfeab, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730993088587 2024-11-07T15:24:50,480 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ac2f402382c47ce99b5b3fbc685d8cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730993088588 2024-11-07T15:24:50,487 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#A#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:50,487 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/80c0956b415e40678b244827bc29b5e6 is 50, key is test_row_0/A:col10/1730993088588/Put/seqid=0 2024-11-07T15:24:50,494 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#B#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:50,495 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/43c9312e9bdd4445a249d9f77b373731 is 50, key is test_row_0/B:col10/1730993088588/Put/seqid=0 2024-11-07T15:24:50,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742307_1483 (size=12697) 2024-11-07T15:24:50,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742308_1484 (size=12697) 2024-11-07T15:24:50,506 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/43c9312e9bdd4445a249d9f77b373731 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/43c9312e9bdd4445a249d9f77b373731 2024-11-07T15:24:50,520 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/B of a4f40fe465370ef7d04dfc7daeebbbb9 into 43c9312e9bdd4445a249d9f77b373731(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:50,521 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:50,521 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/B, priority=13, startTime=1730993090478; duration=0sec 2024-11-07T15:24:50,521 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:50,521 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:B 2024-11-07T15:24:50,521 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:50,521 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:50,521 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/C is initiating minor compaction (all files) 2024-11-07T15:24:50,521 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/C in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:50,521 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/b2bd86e35fa54e34881ba3d22b39b67b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/c0d5cbbfa52a4f9c9507a5b73e30eee5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/6698168d415245598448f2eeae960a6d] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=36.0 K 2024-11-07T15:24:50,522 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting b2bd86e35fa54e34881ba3d22b39b67b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1730993084183 2024-11-07T15:24:50,522 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting c0d5cbbfa52a4f9c9507a5b73e30eee5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730993086369 2024-11-07T15:24:50,522 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 6698168d415245598448f2eeae960a6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730993088588 2024-11-07T15:24:50,530 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#C#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:50,530 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/653c54d6a0cb416da71f882a61bf91e1 is 50, key is test_row_0/C:col10/1730993088588/Put/seqid=0 2024-11-07T15:24:50,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742309_1485 (size=12697) 2024-11-07T15:24:50,541 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/653c54d6a0cb416da71f882a61bf91e1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/653c54d6a0cb416da71f882a61bf91e1 2024-11-07T15:24:50,544 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/C of a4f40fe465370ef7d04dfc7daeebbbb9 into 653c54d6a0cb416da71f882a61bf91e1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:50,544 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:50,544 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/C, priority=13, startTime=1730993090478; duration=0sec 2024-11-07T15:24:50,544 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:50,544 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:C 2024-11-07T15:24:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T15:24:50,625 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:50,625 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-07T15:24:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T15:24:50,627 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:50,628 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:50,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T15:24:50,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T15:24:50,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:50,781 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T15:24:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/40a936ee8516487a8680fbbd7b63582b is 50, key is test_row_0/A:col10/1730993089245/Put/seqid=0 2024-11-07T15:24:50,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742310_1486 (size=12151) 2024-11-07T15:24:50,789 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/40a936ee8516487a8680fbbd7b63582b 2024-11-07T15:24:50,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/cdd3fdde4a8346288c14869dfdab6037 is 50, key is test_row_0/B:col10/1730993089245/Put/seqid=0 2024-11-07T15:24:50,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742311_1487 (size=12151) 2024-11-07T15:24:50,907 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/80c0956b415e40678b244827bc29b5e6 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/80c0956b415e40678b244827bc29b5e6 2024-11-07T15:24:50,910 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/A of a4f40fe465370ef7d04dfc7daeebbbb9 into 80c0956b415e40678b244827bc29b5e6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:50,910 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:50,910 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/A, priority=13, startTime=1730993090478; duration=0sec 2024-11-07T15:24:50,910 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:50,910 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:A 2024-11-07T15:24:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T15:24:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:51,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:51,202 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/cdd3fdde4a8346288c14869dfdab6037 2024-11-07T15:24:51,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/684a8c544bbc4b98bd57bee37ef12c6d is 50, key is test_row_0/C:col10/1730993089245/Put/seqid=0 2024-11-07T15:24:51,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742312_1488 (size=12151) 2024-11-07T15:24:51,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993151205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T15:24:51,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993151321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:51,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993151411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:51,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993151411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:51,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:51,417 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in a4f40fe465370ef7d04dfc7daeebbbb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993151411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:51,418 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_1' in a4f40fe465370ef7d04dfc7daeebbbb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:51,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993151411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:51,418 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_1' in a4f40fe465370ef7d04dfc7daeebbbb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:51,419 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in a4f40fe465370ef7d04dfc7daeebbbb9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:51,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993151531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:51,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:51,609 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/684a8c544bbc4b98bd57bee37ef12c6d 2024-11-07T15:24:51,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/40a936ee8516487a8680fbbd7b63582b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/40a936ee8516487a8680fbbd7b63582b 2024-11-07T15:24:51,615 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/40a936ee8516487a8680fbbd7b63582b, entries=150, sequenceid=245, filesize=11.9 K 2024-11-07T15:24:51,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/cdd3fdde4a8346288c14869dfdab6037 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/cdd3fdde4a8346288c14869dfdab6037 2024-11-07T15:24:51,619 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/cdd3fdde4a8346288c14869dfdab6037, entries=150, sequenceid=245, filesize=11.9 K 2024-11-07T15:24:51,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/684a8c544bbc4b98bd57bee37ef12c6d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/684a8c544bbc4b98bd57bee37ef12c6d 2024-11-07T15:24:51,622 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/684a8c544bbc4b98bd57bee37ef12c6d, entries=150, sequenceid=245, filesize=11.9 K 2024-11-07T15:24:51,623 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for a4f40fe465370ef7d04dfc7daeebbbb9 in 843ms, sequenceid=245, compaction requested=false 2024-11-07T15:24:51,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:51,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:51,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-07T15:24:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-07T15:24:51,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-07T15:24:51,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 996 msec 2024-11-07T15:24:51,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.0010 sec 2024-11-07T15:24:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T15:24:51,761 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:51,761 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-07T15:24:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T15:24:51,763 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:51,763 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:51,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:51,845 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:51,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-07T15:24:51,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/e762476f9a1a4c3f8cc14a6297bee050 is 50, key is test_row_0/A:col10/1730993091179/Put/seqid=0 2024-11-07T15:24:51,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742313_1489 (size=14741) 2024-11-07T15:24:51,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993151865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T15:24:51,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T15:24:51,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:51,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:51,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:51,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:51,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:51,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993151980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:51,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:51,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:52,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T15:24:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:52,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T15:24:52,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993152190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:52,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T15:24:52,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:52,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:52,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:52,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:52,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:52,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/e762476f9a1a4c3f8cc14a6297bee050 2024-11-07T15:24:52,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/4722d1dfafd54509bbf51b77b4764dc8 is 50, key is test_row_0/B:col10/1730993091179/Put/seqid=0 2024-11-07T15:24:52,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742314_1490 (size=12301) 2024-11-07T15:24:52,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T15:24:52,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:52,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:52,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:52,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:52,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T15:24:52,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993152501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:52,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T15:24:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:52,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:52,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/4722d1dfafd54509bbf51b77b4764dc8 2024-11-07T15:24:52,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/c8c29ab2097a49908f07f51af3df4c2a is 50, key is test_row_0/C:col10/1730993091179/Put/seqid=0 2024-11-07T15:24:52,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742315_1491 (size=12301) 2024-11-07T15:24:52,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/c8c29ab2097a49908f07f51af3df4c2a 2024-11-07T15:24:52,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T15:24:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:52,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:52,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:52,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/e762476f9a1a4c3f8cc14a6297bee050 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e762476f9a1a4c3f8cc14a6297bee050 2024-11-07T15:24:52,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e762476f9a1a4c3f8cc14a6297bee050, entries=200, sequenceid=275, filesize=14.4 K 2024-11-07T15:24:52,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/4722d1dfafd54509bbf51b77b4764dc8 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/4722d1dfafd54509bbf51b77b4764dc8 2024-11-07T15:24:52,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/4722d1dfafd54509bbf51b77b4764dc8, entries=150, sequenceid=275, filesize=12.0 K 2024-11-07T15:24:52,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/c8c29ab2097a49908f07f51af3df4c2a as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/c8c29ab2097a49908f07f51af3df4c2a 2024-11-07T15:24:52,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/c8c29ab2097a49908f07f51af3df4c2a, entries=150, sequenceid=275, filesize=12.0 K 2024-11-07T15:24:52,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for a4f40fe465370ef7d04dfc7daeebbbb9 in 856ms, sequenceid=275, compaction requested=true 2024-11-07T15:24:52,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:52,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:24:52,701 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:52,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:52,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:24:52,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:52,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:24:52,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T15:24:52,702 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:52,703 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:52,703 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/B is initiating minor compaction (all files) 2024-11-07T15:24:52,703 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/B in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:52,703 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/43c9312e9bdd4445a249d9f77b373731, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/cdd3fdde4a8346288c14869dfdab6037, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/4722d1dfafd54509bbf51b77b4764dc8] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=36.3 K 2024-11-07T15:24:52,703 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:52,703 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/A is initiating minor compaction (all files) 2024-11-07T15:24:52,704 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/A in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:52,704 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/80c0956b415e40678b244827bc29b5e6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/40a936ee8516487a8680fbbd7b63582b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e762476f9a1a4c3f8cc14a6297bee050] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=38.7 K 2024-11-07T15:24:52,704 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 43c9312e9bdd4445a249d9f77b373731, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730993088588 2024-11-07T15:24:52,704 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80c0956b415e40678b244827bc29b5e6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730993088588 2024-11-07T15:24:52,705 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting cdd3fdde4a8346288c14869dfdab6037, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1730993089237 2024-11-07T15:24:52,706 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40a936ee8516487a8680fbbd7b63582b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1730993089237 2024-11-07T15:24:52,706 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 4722d1dfafd54509bbf51b77b4764dc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1730993091179 2024-11-07T15:24:52,706 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting e762476f9a1a4c3f8cc14a6297bee050, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1730993091179 2024-11-07T15:24:52,714 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#A#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:52,715 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/0c69db268f634788bd37215cee5344bd is 50, key is test_row_0/A:col10/1730993091179/Put/seqid=0 2024-11-07T15:24:52,718 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#B#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:52,718 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/ac4b56fca3e748f6898b6a3c419db22d is 50, key is test_row_0/B:col10/1730993091179/Put/seqid=0 2024-11-07T15:24:52,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742316_1492 (size=12949) 2024-11-07T15:24:52,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742317_1493 (size=12949) 2024-11-07T15:24:52,752 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/0c69db268f634788bd37215cee5344bd as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/0c69db268f634788bd37215cee5344bd 2024-11-07T15:24:52,757 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/ac4b56fca3e748f6898b6a3c419db22d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/ac4b56fca3e748f6898b6a3c419db22d 2024-11-07T15:24:52,761 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/A of a4f40fe465370ef7d04dfc7daeebbbb9 into 0c69db268f634788bd37215cee5344bd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:52,761 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:52,761 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/A, priority=13, startTime=1730993092701; duration=0sec 2024-11-07T15:24:52,761 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:52,761 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:A 2024-11-07T15:24:52,761 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:52,762 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:52,762 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/C is initiating minor compaction (all files) 2024-11-07T15:24:52,762 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/C in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:52,762 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/653c54d6a0cb416da71f882a61bf91e1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/684a8c544bbc4b98bd57bee37ef12c6d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/c8c29ab2097a49908f07f51af3df4c2a] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=36.3 K 2024-11-07T15:24:52,762 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 653c54d6a0cb416da71f882a61bf91e1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730993088588 2024-11-07T15:24:52,763 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 684a8c544bbc4b98bd57bee37ef12c6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1730993089237 2024-11-07T15:24:52,763 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8c29ab2097a49908f07f51af3df4c2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1730993091179 2024-11-07T15:24:52,764 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/B of a4f40fe465370ef7d04dfc7daeebbbb9 into ac4b56fca3e748f6898b6a3c419db22d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:52,764 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:52,764 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/B, priority=13, startTime=1730993092701; duration=0sec 2024-11-07T15:24:52,765 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:52,765 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:B 2024-11-07T15:24:52,770 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#C#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:52,771 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/760cb8e1e8d64d80beee0431b70ff3ac is 50, key is test_row_0/C:col10/1730993091179/Put/seqid=0 2024-11-07T15:24:52,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742318_1494 (size=12949) 2024-11-07T15:24:52,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T15:24:52,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:52,834 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-07T15:24:52,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/a12cd82b94a94780bc49341c30b70079 is 50, key is test_row_0/A:col10/1730993091858/Put/seqid=0 2024-11-07T15:24:52,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742319_1495 (size=9857) 2024-11-07T15:24:52,858 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/a12cd82b94a94780bc49341c30b70079 2024-11-07T15:24:52,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/83e5dc425c1241af897a212e6910e739 is 50, key is test_row_0/B:col10/1730993091858/Put/seqid=0 2024-11-07T15:24:52,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T15:24:52,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742320_1496 (size=9857) 2024-11-07T15:24:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:53,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:53,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993153169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:53,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:53,212 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/760cb8e1e8d64d80beee0431b70ff3ac as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/760cb8e1e8d64d80beee0431b70ff3ac 2024-11-07T15:24:53,218 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/C of a4f40fe465370ef7d04dfc7daeebbbb9 into 760cb8e1e8d64d80beee0431b70ff3ac(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:53,218 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:53,218 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/C, priority=13, startTime=1730993092701; duration=0sec 2024-11-07T15:24:53,218 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:53,218 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:C 2024-11-07T15:24:53,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993153281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:53,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:53,315 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/83e5dc425c1241af897a212e6910e739 2024-11-07T15:24:53,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/722d11984fda4f708019c5e6c321ba3e is 50, key is test_row_0/C:col10/1730993091858/Put/seqid=0 2024-11-07T15:24:53,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742321_1497 (size=9857) 2024-11-07T15:24:53,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:53,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45150 deadline: 1730993153431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:53,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45152 deadline: 1730993153431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:53,437 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_1' in a4f40fe465370ef7d04dfc7daeebbbb9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:53,437 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_1' in a4f40fe465370ef7d04dfc7daeebbbb9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:53,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:53,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45198 deadline: 1730993153440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:53,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:53,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45190 deadline: 1730993153440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:53,447 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in a4f40fe465370ef7d04dfc7daeebbbb9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:53,447 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in a4f40fe465370ef7d04dfc7daeebbbb9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:24:53,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:53,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:53,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:53,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993153491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:53,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:53,725 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/722d11984fda4f708019c5e6c321ba3e 2024-11-07T15:24:53,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/a12cd82b94a94780bc49341c30b70079 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/a12cd82b94a94780bc49341c30b70079 2024-11-07T15:24:53,732 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/a12cd82b94a94780bc49341c30b70079, entries=100, sequenceid=284, filesize=9.6 K 2024-11-07T15:24:53,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/83e5dc425c1241af897a212e6910e739 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/83e5dc425c1241af897a212e6910e739 2024-11-07T15:24:53,741 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/83e5dc425c1241af897a212e6910e739, entries=100, sequenceid=284, filesize=9.6 K 2024-11-07T15:24:53,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/722d11984fda4f708019c5e6c321ba3e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/722d11984fda4f708019c5e6c321ba3e 2024-11-07T15:24:53,747 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/722d11984fda4f708019c5e6c321ba3e, entries=100, sequenceid=284, filesize=9.6 K 2024-11-07T15:24:53,748 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for a4f40fe465370ef7d04dfc7daeebbbb9 in 914ms, sequenceid=284, compaction requested=false 2024-11-07T15:24:53,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:53,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:53,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-07T15:24:53,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-07T15:24:53,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-07T15:24:53,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9870 sec 2024-11-07T15:24:53,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.9950 sec 2024-11-07T15:24:53,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:53,807 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:53,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-07T15:24:53,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/981463c38f564058b76a441df21233de is 50, key is test_row_0/A:col10/1730993093804/Put/seqid=0 2024-11-07T15:24:53,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742322_1498 (size=14741) 2024-11-07T15:24:53,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993153823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:53,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T15:24:53,911 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:53,912 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-07T15:24:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T15:24:53,915 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:53,916 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:53,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:53,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993153931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:53,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:53,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T15:24:54,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:24:54,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:54,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:54,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:54,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:54,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:54,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993154151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:54,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/981463c38f564058b76a441df21233de 2024-11-07T15:24:54,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:24:54,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:54,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:54,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:54,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:54,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T15:24:54,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/eb6ed649fb204e8b90629cbb83346f8f is 50, key is test_row_0/B:col10/1730993093804/Put/seqid=0 2024-11-07T15:24:54,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742323_1499 (size=12301) 2024-11-07T15:24:54,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/eb6ed649fb204e8b90629cbb83346f8f 2024-11-07T15:24:54,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/8f13c114569e415b9baaadf77261afe1 is 50, key is test_row_0/C:col10/1730993093804/Put/seqid=0 2024-11-07T15:24:54,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742324_1500 (size=12301) 2024-11-07T15:24:54,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:24:54,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:54,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:54,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:54,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:54,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:54,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993154460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:54,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:54,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:54,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:54,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:24:54,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:54,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:54,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:54,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:54,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:24:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T15:24:54,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/8f13c114569e415b9baaadf77261afe1 2024-11-07T15:24:54,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/981463c38f564058b76a441df21233de as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/981463c38f564058b76a441df21233de 2024-11-07T15:24:54,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/981463c38f564058b76a441df21233de, entries=200, sequenceid=315, filesize=14.4 K 2024-11-07T15:24:54,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/eb6ed649fb204e8b90629cbb83346f8f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/eb6ed649fb204e8b90629cbb83346f8f 2024-11-07T15:24:54,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/eb6ed649fb204e8b90629cbb83346f8f, entries=150, sequenceid=315, filesize=12.0 K 2024-11-07T15:24:54,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/8f13c114569e415b9baaadf77261afe1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/8f13c114569e415b9baaadf77261afe1 2024-11-07T15:24:54,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/8f13c114569e415b9baaadf77261afe1, entries=150, sequenceid=315, filesize=12.0 K 2024-11-07T15:24:54,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for a4f40fe465370ef7d04dfc7daeebbbb9 in 866ms, sequenceid=315, compaction requested=true 2024-11-07T15:24:54,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:54,674 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:54,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:24:54,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:54,674 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:54,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:24:54,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:54,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:24:54,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:54,676 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:54,676 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37547 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:54,676 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/A is initiating minor compaction (all files) 2024-11-07T15:24:54,676 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/B is initiating minor compaction (all files) 2024-11-07T15:24:54,676 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/A in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:54,676 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/B in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:54,676 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/0c69db268f634788bd37215cee5344bd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/a12cd82b94a94780bc49341c30b70079, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/981463c38f564058b76a441df21233de] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=36.7 K 2024-11-07T15:24:54,676 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/ac4b56fca3e748f6898b6a3c419db22d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/83e5dc425c1241af897a212e6910e739, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/eb6ed649fb204e8b90629cbb83346f8f] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=34.3 K 2024-11-07T15:24:54,677 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting ac4b56fca3e748f6898b6a3c419db22d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1730993091179 2024-11-07T15:24:54,677 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c69db268f634788bd37215cee5344bd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1730993091179 2024-11-07T15:24:54,677 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 83e5dc425c1241af897a212e6910e739, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1730993091858 2024-11-07T15:24:54,677 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting a12cd82b94a94780bc49341c30b70079, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1730993091858 2024-11-07T15:24:54,677 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting eb6ed649fb204e8b90629cbb83346f8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1730993093150 2024-11-07T15:24:54,678 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 981463c38f564058b76a441df21233de, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1730993093150 2024-11-07T15:24:54,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:24:54,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:54,681 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-07T15:24:54,684 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#B#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:54,684 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/835187b2077342679a9e1ded44307352 is 50, key is test_row_0/B:col10/1730993093804/Put/seqid=0 2024-11-07T15:24:54,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/d79b5d8fc2594324a4a9f741777c2741 is 50, key is test_row_0/A:col10/1730993093821/Put/seqid=0 2024-11-07T15:24:54,686 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#A#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:54,686 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/02f705c93cc448fcac77a05e8fd04280 is 50, key is test_row_0/A:col10/1730993093804/Put/seqid=0 2024-11-07T15:24:54,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742325_1501 (size=13051) 2024-11-07T15:24:54,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742326_1502 (size=9857) 2024-11-07T15:24:54,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742327_1503 (size=13051) 2024-11-07T15:24:54,701 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/835187b2077342679a9e1ded44307352 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/835187b2077342679a9e1ded44307352 2024-11-07T15:24:54,701 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/02f705c93cc448fcac77a05e8fd04280 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/02f705c93cc448fcac77a05e8fd04280 2024-11-07T15:24:54,704 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/B of a4f40fe465370ef7d04dfc7daeebbbb9 into 835187b2077342679a9e1ded44307352(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:54,704 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:54,704 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/B, priority=13, startTime=1730993094674; duration=0sec 2024-11-07T15:24:54,704 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:24:54,704 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:B 2024-11-07T15:24:54,704 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:24:54,709 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:24:54,709 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): a4f40fe465370ef7d04dfc7daeebbbb9/C is initiating minor compaction (all files) 2024-11-07T15:24:54,709 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4f40fe465370ef7d04dfc7daeebbbb9/C in TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:54,709 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/760cb8e1e8d64d80beee0431b70ff3ac, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/722d11984fda4f708019c5e6c321ba3e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/8f13c114569e415b9baaadf77261afe1] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp, totalSize=34.3 K 2024-11-07T15:24:54,709 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 760cb8e1e8d64d80beee0431b70ff3ac, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1730993091179 2024-11-07T15:24:54,709 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/A of a4f40fe465370ef7d04dfc7daeebbbb9 into 02f705c93cc448fcac77a05e8fd04280(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:54,710 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:54,710 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/A, priority=13, startTime=1730993094674; duration=0sec 2024-11-07T15:24:54,710 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:54,710 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:A 2024-11-07T15:24:54,710 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 722d11984fda4f708019c5e6c321ba3e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1730993091858 2024-11-07T15:24:54,712 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f13c114569e415b9baaadf77261afe1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1730993093150 2024-11-07T15:24:54,718 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4f40fe465370ef7d04dfc7daeebbbb9#C#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:24:54,719 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/9d24cd098d6142b190f376402c48154c is 50, key is test_row_0/C:col10/1730993093804/Put/seqid=0 2024-11-07T15:24:54,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742328_1504 (size=13051) 2024-11-07T15:24:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:55,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T15:24:55,097 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/d79b5d8fc2594324a4a9f741777c2741 2024-11-07T15:24:55,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/c5319c6150b34f4780f9c020ddfc7012 is 50, key is test_row_0/B:col10/1730993093821/Put/seqid=0 2024-11-07T15:24:55,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742329_1505 (size=9857) 2024-11-07T15:24:55,131 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/9d24cd098d6142b190f376402c48154c as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/9d24cd098d6142b190f376402c48154c 2024-11-07T15:24:55,135 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4f40fe465370ef7d04dfc7daeebbbb9/C of a4f40fe465370ef7d04dfc7daeebbbb9 into 9d24cd098d6142b190f376402c48154c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:24:55,135 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:55,135 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., storeName=a4f40fe465370ef7d04dfc7daeebbbb9/C, priority=13, startTime=1730993094674; duration=0sec 2024-11-07T15:24:55,135 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:55,135 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:C 2024-11-07T15:24:55,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993155154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:55,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993155271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:55,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:55,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:55,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:55,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993155481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:55,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:55,524 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/c5319c6150b34f4780f9c020ddfc7012 2024-11-07T15:24:55,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/b97d3c2aaac0478fbc1a3334bf95061d is 50, key is test_row_0/C:col10/1730993093821/Put/seqid=0 2024-11-07T15:24:55,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742330_1506 (size=9857) 2024-11-07T15:24:55,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993155801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:55,933 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/b97d3c2aaac0478fbc1a3334bf95061d 2024-11-07T15:24:55,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/d79b5d8fc2594324a4a9f741777c2741 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/d79b5d8fc2594324a4a9f741777c2741 2024-11-07T15:24:55,940 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/d79b5d8fc2594324a4a9f741777c2741, entries=100, sequenceid=321, filesize=9.6 K 2024-11-07T15:24:55,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/c5319c6150b34f4780f9c020ddfc7012 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/c5319c6150b34f4780f9c020ddfc7012 2024-11-07T15:24:55,946 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/c5319c6150b34f4780f9c020ddfc7012, entries=100, sequenceid=321, filesize=9.6 K 2024-11-07T15:24:55,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/b97d3c2aaac0478fbc1a3334bf95061d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/b97d3c2aaac0478fbc1a3334bf95061d 2024-11-07T15:24:55,951 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/b97d3c2aaac0478fbc1a3334bf95061d, entries=100, sequenceid=321, filesize=9.6 K 2024-11-07T15:24:55,951 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for a4f40fe465370ef7d04dfc7daeebbbb9 in 1270ms, sequenceid=321, compaction requested=false 2024-11-07T15:24:55,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:55,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:55,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-07T15:24:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-07T15:24:55,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-07T15:24:55,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0360 sec 2024-11-07T15:24:55,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.0420 sec 2024-11-07T15:24:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T15:24:56,061 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:56,062 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:24:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-07T15:24:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T15:24:56,063 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:24:56,064 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:24:56,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:24:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T15:24:56,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-07T15:24:56,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:56,216 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-07T15:24:56,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/3095f0eb39434b57b70a6d51b4b24c0f is 50, key is test_row_0/A:col10/1730993095152/Put/seqid=0 2024-11-07T15:24:56,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742331_1507 (size=12301) 2024-11-07T15:24:56,225 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/3095f0eb39434b57b70a6d51b4b24c0f 2024-11-07T15:24:56,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/d4f1a0c652d34a15a06b6ad7ab331042 is 50, key is test_row_0/B:col10/1730993095152/Put/seqid=0 2024-11-07T15:24:56,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742332_1508 (size=12301) 2024-11-07T15:24:56,236 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/d4f1a0c652d34a15a06b6ad7ab331042 2024-11-07T15:24:56,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/11ea125357d944f28369e98adee2ea2d is 50, key is test_row_0/C:col10/1730993095152/Put/seqid=0 2024-11-07T15:24:56,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742333_1509 (size=12301) 2024-11-07T15:24:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:56,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. as already flushing 2024-11-07T15:24:56,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993156321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:56,356 INFO [Thread-1964 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1964. 2024-11-07T15:24:56,356 DEBUG [Thread-1964 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:56,356 DEBUG [Thread-1964 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:56,356 DEBUG [Thread-1964 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:56,356 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:56,358 INFO [Thread-1966 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1966. 2024-11-07T15:24:56,358 DEBUG [Thread-1966 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:56,358 DEBUG [Thread-1966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:56,358 DEBUG [Thread-1966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:56,358 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:56,358 INFO [Thread-1968 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1968. 2024-11-07T15:24:56,358 DEBUG [Thread-1968 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:56,358 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:56,358 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:56,358 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:56,363 INFO [Thread-1970 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1970. 2024-11-07T15:24:56,363 DEBUG [Thread-1970 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:56,363 DEBUG [Thread-1970 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:56,363 DEBUG [Thread-1970 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:56,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:56,364 INFO [Thread-1962 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1962. 2024-11-07T15:24:56,364 DEBUG [Thread-1962 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:56,364 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:56,364 DEBUG [Thread-1962 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:56,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T15:24:56,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993156431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:56,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:56,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:24:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1730993156641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:56,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 , the old value is region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:56,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:24:56,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:24:56,647 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/11ea125357d944f28369e98adee2ea2d 2024-11-07T15:24:56,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/3095f0eb39434b57b70a6d51b4b24c0f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/3095f0eb39434b57b70a6d51b4b24c0f 2024-11-07T15:24:56,652 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/3095f0eb39434b57b70a6d51b4b24c0f, entries=150, sequenceid=354, filesize=12.0 K 2024-11-07T15:24:56,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/d4f1a0c652d34a15a06b6ad7ab331042 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/d4f1a0c652d34a15a06b6ad7ab331042 2024-11-07T15:24:56,655 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/d4f1a0c652d34a15a06b6ad7ab331042, entries=150, sequenceid=354, filesize=12.0 K 2024-11-07T15:24:56,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/11ea125357d944f28369e98adee2ea2d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/11ea125357d944f28369e98adee2ea2d 2024-11-07T15:24:56,657 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/11ea125357d944f28369e98adee2ea2d, entries=150, sequenceid=354, filesize=12.0 K 2024-11-07T15:24:56,658 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for a4f40fe465370ef7d04dfc7daeebbbb9 in 442ms, sequenceid=354, compaction requested=true 2024-11-07T15:24:56,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:56,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:56,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-07T15:24:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-07T15:24:56,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-07T15:24:56,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 595 msec 2024-11-07T15:24:56,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 599 msec 2024-11-07T15:24:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T15:24:56,690 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:56,952 INFO [Thread-1959 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1959. 2024-11-07T15:24:56,952 DEBUG [Thread-1959 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:56,952 DEBUG [Thread-1959 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:56,952 DEBUG [Thread-1959 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:56,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:57,452 INFO [Thread-1953 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1953. 2024-11-07T15:24:57,452 DEBUG [Thread-1953 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:57,452 DEBUG [Thread-1953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:57,452 DEBUG [Thread-1953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:57,452 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:57,462 INFO [Thread-1951 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1951. 2024-11-07T15:24:57,462 INFO [Thread-1955 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1955. 2024-11-07T15:24:57,462 DEBUG [Thread-1955 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:57,462 DEBUG [Thread-1951 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:57,462 DEBUG [Thread-1955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:57,462 DEBUG [Thread-1951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:57,462 DEBUG [Thread-1955 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:57,462 DEBUG [Thread-1951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:57,462 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:57,462 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:57,472 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:24:57,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4f40fe465370ef7d04dfc7daeebbbb9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:24:57,472 INFO [Thread-1957 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1957. 2024-11-07T15:24:57,472 DEBUG [Thread-1957 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:24:57,472 DEBUG [Thread-1957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:57,472 DEBUG [Thread-1957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:57,472 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1694 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5081 rows 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1697 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5087 rows 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1698 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5088 rows 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1691 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5069 rows 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1694 2024-11-07T15:24:57,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5082 rows 2024-11-07T15:24:57,473 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T15:24:57,473 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:24:57,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:57,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:24:57,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:24:57,474 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T15:24:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:57,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/0695f288398f4eeaa39bd5251fa4bc1a is 50, key is test_row_0/A:col10/1730993096317/Put/seqid=0 2024-11-07T15:24:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T15:24:57,476 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993097476"}]},"ts":"1730993097476"} 2024-11-07T15:24:57,477 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T15:24:57,477 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T15:24:57,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:24:57,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742334_1510 (size=12301) 2024-11-07T15:24:57,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4f40fe465370ef7d04dfc7daeebbbb9, UNASSIGN}] 2024-11-07T15:24:57,480 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4f40fe465370ef7d04dfc7daeebbbb9, UNASSIGN 2024-11-07T15:24:57,480 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=a4f40fe465370ef7d04dfc7daeebbbb9, regionState=CLOSING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:24:57,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4f40fe465370ef7d04dfc7daeebbbb9, UNASSIGN because future has completed 2024-11-07T15:24:57,482 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:24:57,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:24:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T15:24:57,634 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:57,634 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:24:57,634 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing a4f40fe465370ef7d04dfc7daeebbbb9, disabling compactions & flushes 2024-11-07T15:24:57,634 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:57,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T15:24:57,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/0695f288398f4eeaa39bd5251fa4bc1a 2024-11-07T15:24:57,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/4b58bdf173944bb79a63ed385cba9ef1 is 50, key is test_row_0/B:col10/1730993096317/Put/seqid=0 2024-11-07T15:24:57,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742335_1511 (size=12301) 2024-11-07T15:24:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T15:24:58,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/4b58bdf173944bb79a63ed385cba9ef1 2024-11-07T15:24:58,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/1d304b03e7404a728b06255f09508722 is 50, key is test_row_0/C:col10/1730993096317/Put/seqid=0 2024-11-07T15:24:58,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742336_1512 (size=12301) 2024-11-07T15:24:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T15:24:58,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/1d304b03e7404a728b06255f09508722 2024-11-07T15:24:58,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/A/0695f288398f4eeaa39bd5251fa4bc1a as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/0695f288398f4eeaa39bd5251fa4bc1a 2024-11-07T15:24:58,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/0695f288398f4eeaa39bd5251fa4bc1a, entries=150, sequenceid=365, filesize=12.0 K 2024-11-07T15:24:58,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/B/4b58bdf173944bb79a63ed385cba9ef1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/4b58bdf173944bb79a63ed385cba9ef1 2024-11-07T15:24:58,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/4b58bdf173944bb79a63ed385cba9ef1, entries=150, sequenceid=365, filesize=12.0 K 2024-11-07T15:24:58,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/.tmp/C/1d304b03e7404a728b06255f09508722 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/1d304b03e7404a728b06255f09508722 2024-11-07T15:24:58,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/1d304b03e7404a728b06255f09508722, entries=150, sequenceid=365, filesize=12.0 K 2024-11-07T15:24:58,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for a4f40fe465370ef7d04dfc7daeebbbb9 in 1234ms, sequenceid=365, compaction requested=true 2024-11-07T15:24:58,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4f40fe465370ef7d04dfc7daeebbbb9: 2024-11-07T15:24:58,706 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:58,706 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:58,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:24:58,706 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. because compaction request was cancelled 2024-11-07T15:24:58,706 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. after waiting 0 ms 2024-11-07T15:24:58,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:58,706 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:58,706 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:A 2024-11-07T15:24:58,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:24:58,706 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. because compaction request was cancelled 2024-11-07T15:24:58,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:58,706 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:B 2024-11-07T15:24:58,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4f40fe465370ef7d04dfc7daeebbbb9:C, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:24:58,706 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. because compaction request was cancelled 2024-11-07T15:24:58,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:24:58,706 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4f40fe465370ef7d04dfc7daeebbbb9:C 2024-11-07T15:24:58,706 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/77a9bfbf41134415bafaf4dba9c8c99f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/b050a2ffe51444aa9624d93096448463, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e43e78ee97334a758a554e075aa79270, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/fcfabbd3779a4ed4a7bfabc6ad913df4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/df6ffa94f965454f900fd1b0b1cf8d8c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/9f4a658f0ab64571a6ee026a6b1d32b5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e125febe38014b4fa403efa82a104729, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/f86f0f3868e240c5996fbb9aa599a9f3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/b9aed944c39841aeabec4c55696e2704, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/7fa64416106e44bd9b17d7396dacf01b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/4cea5b1924274a779b624552b2f62588, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/c15a9a8ff5ed48159cf3a2daab4553f6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/25d394eaf82d443793659e60ca8247f6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/85b1318de88d4157a99d6b1283d1dc34, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/dc5f789b976c48cca127ddc592391b67, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/f1e8d4f1050d48ac94c126a96cbdfeab, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/80c0956b415e40678b244827bc29b5e6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/40a936ee8516487a8680fbbd7b63582b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e762476f9a1a4c3f8cc14a6297bee050, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/0c69db268f634788bd37215cee5344bd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/a12cd82b94a94780bc49341c30b70079, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/981463c38f564058b76a441df21233de] to archive 2024-11-07T15:24:58,707 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:24:58,708 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/77a9bfbf41134415bafaf4dba9c8c99f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/77a9bfbf41134415bafaf4dba9c8c99f 2024-11-07T15:24:58,709 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/b050a2ffe51444aa9624d93096448463 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/b050a2ffe51444aa9624d93096448463 2024-11-07T15:24:58,709 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e43e78ee97334a758a554e075aa79270 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e43e78ee97334a758a554e075aa79270 2024-11-07T15:24:58,710 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/fcfabbd3779a4ed4a7bfabc6ad913df4 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/fcfabbd3779a4ed4a7bfabc6ad913df4 2024-11-07T15:24:58,711 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/df6ffa94f965454f900fd1b0b1cf8d8c to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/df6ffa94f965454f900fd1b0b1cf8d8c 2024-11-07T15:24:58,712 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/9f4a658f0ab64571a6ee026a6b1d32b5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/9f4a658f0ab64571a6ee026a6b1d32b5 2024-11-07T15:24:58,712 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e125febe38014b4fa403efa82a104729 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e125febe38014b4fa403efa82a104729 2024-11-07T15:24:58,713 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/f86f0f3868e240c5996fbb9aa599a9f3 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/f86f0f3868e240c5996fbb9aa599a9f3 2024-11-07T15:24:58,714 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/b9aed944c39841aeabec4c55696e2704 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/b9aed944c39841aeabec4c55696e2704 2024-11-07T15:24:58,715 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/7fa64416106e44bd9b17d7396dacf01b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/7fa64416106e44bd9b17d7396dacf01b 2024-11-07T15:24:58,715 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/4cea5b1924274a779b624552b2f62588 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/4cea5b1924274a779b624552b2f62588 2024-11-07T15:24:58,716 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/c15a9a8ff5ed48159cf3a2daab4553f6 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/c15a9a8ff5ed48159cf3a2daab4553f6 2024-11-07T15:24:58,717 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/25d394eaf82d443793659e60ca8247f6 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/25d394eaf82d443793659e60ca8247f6 2024-11-07T15:24:58,717 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/85b1318de88d4157a99d6b1283d1dc34 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/85b1318de88d4157a99d6b1283d1dc34 2024-11-07T15:24:58,718 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/dc5f789b976c48cca127ddc592391b67 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/dc5f789b976c48cca127ddc592391b67 2024-11-07T15:24:58,719 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/f1e8d4f1050d48ac94c126a96cbdfeab to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/f1e8d4f1050d48ac94c126a96cbdfeab 2024-11-07T15:24:58,719 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/80c0956b415e40678b244827bc29b5e6 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/80c0956b415e40678b244827bc29b5e6 2024-11-07T15:24:58,720 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/40a936ee8516487a8680fbbd7b63582b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/40a936ee8516487a8680fbbd7b63582b 2024-11-07T15:24:58,721 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e762476f9a1a4c3f8cc14a6297bee050 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/e762476f9a1a4c3f8cc14a6297bee050 2024-11-07T15:24:58,721 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/0c69db268f634788bd37215cee5344bd to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/0c69db268f634788bd37215cee5344bd 2024-11-07T15:24:58,722 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/a12cd82b94a94780bc49341c30b70079 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/a12cd82b94a94780bc49341c30b70079 2024-11-07T15:24:58,723 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/981463c38f564058b76a441df21233de to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/981463c38f564058b76a441df21233de 2024-11-07T15:24:58,724 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/82be4ffc74bf4b858aa57d8507858178, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/9565d81f5cbc4a038be929f54800185e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/e29390e07db647d4891f7d4e5351015f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/40d75a7ce5614dfe92834e32ca7be074, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/8fab5e8462da451595bd5e4cdd3be3fe, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/84403c51bd1f479592a6d2df659891f0, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/f0cbc1c656274c91a505b256081f7078, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/72de5b845fbd4e5490e4b1703ef6206a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/7069b470a7f34799a4f311a8ae0326be, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/ac0807dd4de042b1a9cbecd38d1a91e1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/47259e67e4be4c29b6ecfbaaa23a23c2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/6d37c09790b54409b825b47892d45b30, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/bd379d613c204a8a84413875340ffc92, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/238b888fb8c7443f815d94384d6667a8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/9340c18b3d414225a1ce68762d96cad1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/43c9312e9bdd4445a249d9f77b373731, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/3ac2f402382c47ce99b5b3fbc685d8cd, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/cdd3fdde4a8346288c14869dfdab6037, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/ac4b56fca3e748f6898b6a3c419db22d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/4722d1dfafd54509bbf51b77b4764dc8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/83e5dc425c1241af897a212e6910e739, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/eb6ed649fb204e8b90629cbb83346f8f] to archive 2024-11-07T15:24:58,724 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:24:58,725 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/82be4ffc74bf4b858aa57d8507858178 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/82be4ffc74bf4b858aa57d8507858178 2024-11-07T15:24:58,726 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/9565d81f5cbc4a038be929f54800185e to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/9565d81f5cbc4a038be929f54800185e 2024-11-07T15:24:58,726 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/e29390e07db647d4891f7d4e5351015f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/e29390e07db647d4891f7d4e5351015f 2024-11-07T15:24:58,727 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/40d75a7ce5614dfe92834e32ca7be074 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/40d75a7ce5614dfe92834e32ca7be074 2024-11-07T15:24:58,727 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/8fab5e8462da451595bd5e4cdd3be3fe to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/8fab5e8462da451595bd5e4cdd3be3fe 2024-11-07T15:24:58,728 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/84403c51bd1f479592a6d2df659891f0 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/84403c51bd1f479592a6d2df659891f0 2024-11-07T15:24:58,729 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/f0cbc1c656274c91a505b256081f7078 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/f0cbc1c656274c91a505b256081f7078 2024-11-07T15:24:58,729 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/72de5b845fbd4e5490e4b1703ef6206a to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/72de5b845fbd4e5490e4b1703ef6206a 2024-11-07T15:24:58,730 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/7069b470a7f34799a4f311a8ae0326be to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/7069b470a7f34799a4f311a8ae0326be 2024-11-07T15:24:58,731 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/ac0807dd4de042b1a9cbecd38d1a91e1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/ac0807dd4de042b1a9cbecd38d1a91e1 2024-11-07T15:24:58,731 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/47259e67e4be4c29b6ecfbaaa23a23c2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/47259e67e4be4c29b6ecfbaaa23a23c2 2024-11-07T15:24:58,732 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/6d37c09790b54409b825b47892d45b30 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/6d37c09790b54409b825b47892d45b30 2024-11-07T15:24:58,733 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/bd379d613c204a8a84413875340ffc92 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/bd379d613c204a8a84413875340ffc92 2024-11-07T15:24:58,733 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/238b888fb8c7443f815d94384d6667a8 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/238b888fb8c7443f815d94384d6667a8 2024-11-07T15:24:58,734 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/9340c18b3d414225a1ce68762d96cad1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/9340c18b3d414225a1ce68762d96cad1 2024-11-07T15:24:58,734 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/43c9312e9bdd4445a249d9f77b373731 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/43c9312e9bdd4445a249d9f77b373731 2024-11-07T15:24:58,735 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/3ac2f402382c47ce99b5b3fbc685d8cd to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/3ac2f402382c47ce99b5b3fbc685d8cd 2024-11-07T15:24:58,736 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/cdd3fdde4a8346288c14869dfdab6037 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/cdd3fdde4a8346288c14869dfdab6037 2024-11-07T15:24:58,736 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/ac4b56fca3e748f6898b6a3c419db22d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/ac4b56fca3e748f6898b6a3c419db22d 2024-11-07T15:24:58,737 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/4722d1dfafd54509bbf51b77b4764dc8 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/4722d1dfafd54509bbf51b77b4764dc8 2024-11-07T15:24:58,738 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/83e5dc425c1241af897a212e6910e739 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/83e5dc425c1241af897a212e6910e739 2024-11-07T15:24:58,738 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/eb6ed649fb204e8b90629cbb83346f8f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/eb6ed649fb204e8b90629cbb83346f8f 2024-11-07T15:24:58,739 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/6af164766da74fe094cbeb37c741c026, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/23ced06617c649978ed0fb7553cb7730, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/030fa26683fe4fbfaf17c2050011d060, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2b6c471cc2234612ad38670d62d8d838, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/1527f1fc55ed4074bf2be58df1ed1088, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2fbde78f8a024687b98602bb6d05acf2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/8d91262b1f2f491094fb0e17960cb0c3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/dd9abfdb37134957aeb1aad8ff9aadc4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/123912095ecc467397366e6c5d28eaf3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/771866541f8744a9b2f7728d08625316, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/a504c00140b640c4833fae38df977bfb, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2803fcac15da4f2c86c2f2b8dcdd80ec, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/b2bd86e35fa54e34881ba3d22b39b67b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/49cffa281d094c75be214486f89ef174, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/c0d5cbbfa52a4f9c9507a5b73e30eee5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/653c54d6a0cb416da71f882a61bf91e1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/6698168d415245598448f2eeae960a6d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/684a8c544bbc4b98bd57bee37ef12c6d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/760cb8e1e8d64d80beee0431b70ff3ac, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/c8c29ab2097a49908f07f51af3df4c2a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/722d11984fda4f708019c5e6c321ba3e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/8f13c114569e415b9baaadf77261afe1] to archive 2024-11-07T15:24:58,740 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:24:58,741 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/6af164766da74fe094cbeb37c741c026 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/6af164766da74fe094cbeb37c741c026 2024-11-07T15:24:58,741 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/23ced06617c649978ed0fb7553cb7730 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/23ced06617c649978ed0fb7553cb7730 2024-11-07T15:24:58,742 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/030fa26683fe4fbfaf17c2050011d060 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/030fa26683fe4fbfaf17c2050011d060 2024-11-07T15:24:58,743 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2b6c471cc2234612ad38670d62d8d838 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2b6c471cc2234612ad38670d62d8d838 2024-11-07T15:24:58,743 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/1527f1fc55ed4074bf2be58df1ed1088 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/1527f1fc55ed4074bf2be58df1ed1088 2024-11-07T15:24:58,744 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2fbde78f8a024687b98602bb6d05acf2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2fbde78f8a024687b98602bb6d05acf2 2024-11-07T15:24:58,744 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/8d91262b1f2f491094fb0e17960cb0c3 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/8d91262b1f2f491094fb0e17960cb0c3 2024-11-07T15:24:58,745 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/dd9abfdb37134957aeb1aad8ff9aadc4 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/dd9abfdb37134957aeb1aad8ff9aadc4 2024-11-07T15:24:58,746 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/123912095ecc467397366e6c5d28eaf3 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/123912095ecc467397366e6c5d28eaf3 2024-11-07T15:24:58,746 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/771866541f8744a9b2f7728d08625316 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/771866541f8744a9b2f7728d08625316 2024-11-07T15:24:58,747 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/a504c00140b640c4833fae38df977bfb to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/a504c00140b640c4833fae38df977bfb 2024-11-07T15:24:58,748 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2803fcac15da4f2c86c2f2b8dcdd80ec to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/2803fcac15da4f2c86c2f2b8dcdd80ec 2024-11-07T15:24:58,749 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/b2bd86e35fa54e34881ba3d22b39b67b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/b2bd86e35fa54e34881ba3d22b39b67b 2024-11-07T15:24:58,749 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/49cffa281d094c75be214486f89ef174 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/49cffa281d094c75be214486f89ef174 2024-11-07T15:24:58,750 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/c0d5cbbfa52a4f9c9507a5b73e30eee5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/c0d5cbbfa52a4f9c9507a5b73e30eee5 2024-11-07T15:24:58,751 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/653c54d6a0cb416da71f882a61bf91e1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/653c54d6a0cb416da71f882a61bf91e1 2024-11-07T15:24:58,751 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/6698168d415245598448f2eeae960a6d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/6698168d415245598448f2eeae960a6d 2024-11-07T15:24:58,752 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/684a8c544bbc4b98bd57bee37ef12c6d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/684a8c544bbc4b98bd57bee37ef12c6d 2024-11-07T15:24:58,753 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/760cb8e1e8d64d80beee0431b70ff3ac to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/760cb8e1e8d64d80beee0431b70ff3ac 2024-11-07T15:24:58,753 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/c8c29ab2097a49908f07f51af3df4c2a to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/c8c29ab2097a49908f07f51af3df4c2a 2024-11-07T15:24:58,754 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/722d11984fda4f708019c5e6c321ba3e to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/722d11984fda4f708019c5e6c321ba3e 2024-11-07T15:24:58,755 DEBUG [StoreCloser-TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/8f13c114569e415b9baaadf77261afe1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/8f13c114569e415b9baaadf77261afe1 2024-11-07T15:24:58,759 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/recovered.edits/368.seqid, newMaxSeqId=368, maxSeqId=1 2024-11-07T15:24:58,759 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9. 2024-11-07T15:24:58,759 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for a4f40fe465370ef7d04dfc7daeebbbb9: Waiting for close lock at 1730993097634Running coprocessor pre-close hooks at 1730993097634Disabling compacts and flushes for region at 1730993097634Disabling writes for close at 1730993098706 (+1072 ms)Writing region close event to WAL at 1730993098756 (+50 ms)Running coprocessor post-close hooks at 1730993098759 (+3 ms)Closed at 1730993098759 2024-11-07T15:24:58,760 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:58,761 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=a4f40fe465370ef7d04dfc7daeebbbb9, regionState=CLOSED 2024-11-07T15:24:58,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:24:58,764 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-07T15:24:58,764 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure a4f40fe465370ef7d04dfc7daeebbbb9, server=7b3e4e8e12ef,33849,1730992951906 in 1.2800 sec 2024-11-07T15:24:58,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-07T15:24:58,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4f40fe465370ef7d04dfc7daeebbbb9, UNASSIGN in 1.2850 sec 2024-11-07T15:24:58,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-07T15:24:58,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2870 sec 2024-11-07T15:24:58,767 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993098767"}]},"ts":"1730993098767"} 2024-11-07T15:24:58,768 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T15:24:58,768 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T15:24:58,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2950 sec 2024-11-07T15:24:59,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-07T15:24:59,631 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:59,631 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T15:24:59,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:59,633 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:59,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T15:24:59,633 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:59,635 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:59,637 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/recovered.edits] 2024-11-07T15:24:59,639 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/02f705c93cc448fcac77a05e8fd04280 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/02f705c93cc448fcac77a05e8fd04280 2024-11-07T15:24:59,640 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/0695f288398f4eeaa39bd5251fa4bc1a to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/0695f288398f4eeaa39bd5251fa4bc1a 2024-11-07T15:24:59,640 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/3095f0eb39434b57b70a6d51b4b24c0f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/3095f0eb39434b57b70a6d51b4b24c0f 2024-11-07T15:24:59,641 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/d79b5d8fc2594324a4a9f741777c2741 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/A/d79b5d8fc2594324a4a9f741777c2741 2024-11-07T15:24:59,643 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/4b58bdf173944bb79a63ed385cba9ef1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/4b58bdf173944bb79a63ed385cba9ef1 2024-11-07T15:24:59,643 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/835187b2077342679a9e1ded44307352 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/835187b2077342679a9e1ded44307352 2024-11-07T15:24:59,644 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/c5319c6150b34f4780f9c020ddfc7012 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/c5319c6150b34f4780f9c020ddfc7012 2024-11-07T15:24:59,645 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/d4f1a0c652d34a15a06b6ad7ab331042 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/B/d4f1a0c652d34a15a06b6ad7ab331042 2024-11-07T15:24:59,646 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/11ea125357d944f28369e98adee2ea2d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/11ea125357d944f28369e98adee2ea2d 2024-11-07T15:24:59,647 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/1d304b03e7404a728b06255f09508722 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/1d304b03e7404a728b06255f09508722 2024-11-07T15:24:59,648 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/9d24cd098d6142b190f376402c48154c to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/9d24cd098d6142b190f376402c48154c 2024-11-07T15:24:59,649 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/b97d3c2aaac0478fbc1a3334bf95061d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/C/b97d3c2aaac0478fbc1a3334bf95061d 2024-11-07T15:24:59,650 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/recovered.edits/368.seqid to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9/recovered.edits/368.seqid 2024-11-07T15:24:59,651 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/a4f40fe465370ef7d04dfc7daeebbbb9 2024-11-07T15:24:59,651 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T15:24:59,652 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:59,654 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T15:24:59,656 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T15:24:59,657 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:59,657 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T15:24:59,657 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730993099657"}]},"ts":"9223372036854775807"} 2024-11-07T15:24:59,658 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T15:24:59,658 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a4f40fe465370ef7d04dfc7daeebbbb9, NAME => 'TestAcidGuarantees,,1730993074136.a4f40fe465370ef7d04dfc7daeebbbb9.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T15:24:59,659 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T15:24:59,659 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730993099659"}]},"ts":"9223372036854775807"} 2024-11-07T15:24:59,660 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T15:24:59,661 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:59,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-07T15:24:59,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-07T15:24:59,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T15:24:59,744 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:24:59,750 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 237) - Thread LEAK? -, OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=395 (was 438), ProcessCount=11 (was 11), AvailableMemoryMB=1224 (was 1249) 2024-11-07T15:24:59,757 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=395, ProcessCount=11, AvailableMemoryMB=1223 2024-11-07T15:24:59,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:24:59,759 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:24:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T15:24:59,761 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T15:24:59,761 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:24:59,761 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-11-07T15:24:59,761 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T15:24:59,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-07T15:24:59,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742337_1513 (size=959) 2024-11-07T15:24:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-07T15:25:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-07T15:25:00,167 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938 2024-11-07T15:25:00,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742338_1514 (size=53) 2024-11-07T15:25:00,293 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T15:25:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-07T15:25:00,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:25:00,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5489735189eabe07c0ddbb2d15b5812f, disabling compactions & flushes 2024-11-07T15:25:00,572 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:00,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:00,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. after waiting 0 ms 2024-11-07T15:25:00,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:00,572 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:00,572 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5489735189eabe07c0ddbb2d15b5812f: Waiting for close lock at 1730993100572Disabling compacts and flushes for region at 1730993100572Disabling writes for close at 1730993100572Writing region close event to WAL at 1730993100572Closed at 1730993100572 2024-11-07T15:25:00,573 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T15:25:00,573 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730993100573"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730993100573"}]},"ts":"1730993100573"} 2024-11-07T15:25:00,574 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T15:25:00,575 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T15:25:00,575 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993100575"}]},"ts":"1730993100575"} 2024-11-07T15:25:00,576 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T15:25:00,576 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5489735189eabe07c0ddbb2d15b5812f, ASSIGN}] 2024-11-07T15:25:00,577 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5489735189eabe07c0ddbb2d15b5812f, ASSIGN 2024-11-07T15:25:00,577 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5489735189eabe07c0ddbb2d15b5812f, ASSIGN; state=OFFLINE, location=7b3e4e8e12ef,33849,1730992951906; forceNewPlan=false, retain=false 2024-11-07T15:25:00,728 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=5489735189eabe07c0ddbb2d15b5812f, regionState=OPENING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:00,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5489735189eabe07c0ddbb2d15b5812f, ASSIGN because future has completed 2024-11-07T15:25:00,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:25:00,883 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:00,884 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:25:00,884 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:00,884 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:25:00,884 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:00,884 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:00,885 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:00,886 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5489735189eabe07c0ddbb2d15b5812f columnFamilyName A 2024-11-07T15:25:00,886 DEBUG [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:00,887 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] regionserver.HStore(327): Store=5489735189eabe07c0ddbb2d15b5812f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:25:00,887 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:00,887 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5489735189eabe07c0ddbb2d15b5812f columnFamilyName B 2024-11-07T15:25:00,888 DEBUG [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:00,888 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] regionserver.HStore(327): Store=5489735189eabe07c0ddbb2d15b5812f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:25:00,888 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:00,889 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5489735189eabe07c0ddbb2d15b5812f columnFamilyName C 2024-11-07T15:25:00,889 DEBUG [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:00,889 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] regionserver.HStore(327): Store=5489735189eabe07c0ddbb2d15b5812f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:25:00,889 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:00,890 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:00,890 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:00,891 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:00,891 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:00,891 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:25:00,892 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:00,893 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T15:25:00,893 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 5489735189eabe07c0ddbb2d15b5812f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66055348, jitterRate=-0.015698611736297607}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:25:00,894 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:00,894 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 5489735189eabe07c0ddbb2d15b5812f: Running coprocessor pre-open hook at 1730993100884Writing region info on filesystem at 1730993100884Initializing all the Stores at 1730993100885 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993100885Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993100885Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993100885Cleaning up temporary data from old regions at 1730993100891 (+6 ms)Running coprocessor post-open hooks at 1730993100894 (+3 ms)Region opened successfully at 1730993100894 2024-11-07T15:25:00,895 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., pid=146, masterSystemTime=1730993100881 2024-11-07T15:25:00,896 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:00,896 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:00,896 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=5489735189eabe07c0ddbb2d15b5812f, regionState=OPEN, openSeqNum=2, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:00,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:25:00,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-07T15:25:00,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 in 169 msec 2024-11-07T15:25:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-07T15:25:00,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-07T15:25:00,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5489735189eabe07c0ddbb2d15b5812f, ASSIGN in 323 msec 2024-11-07T15:25:00,901 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T15:25:00,902 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993100901"}]},"ts":"1730993100901"} 2024-11-07T15:25:00,903 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T15:25:00,903 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T15:25:00,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1440 sec 2024-11-07T15:25:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-07T15:25:01,912 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:25:01,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64082ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:01,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:25:01,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:25:01,913 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:25:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:25:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:25:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e94f29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:25:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:25:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:01,915 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:25:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b3a720, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:01,916 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:25:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:25:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:25:01,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:25:01,918 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T15:25:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b3e4e8e12ef,46811,1730992951165 2024-11-07T15:25:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2a7217c7 2024-11-07T15:25:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T15:25:01,919 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T15:25:01,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:25:01,921 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:25:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T15:25:01,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742339_1515 (size=995) 2024-11-07T15:25:02,331 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T15:25:02,331 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T15:25:02,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:25:02,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5489735189eabe07c0ddbb2d15b5812f, REOPEN/MOVE}] 2024-11-07T15:25:02,335 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5489735189eabe07c0ddbb2d15b5812f, REOPEN/MOVE 2024-11-07T15:25:02,336 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=5489735189eabe07c0ddbb2d15b5812f, regionState=CLOSING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:02,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5489735189eabe07c0ddbb2d15b5812f, REOPEN/MOVE because future has completed 2024-11-07T15:25:02,337 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:25:02,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:25:02,489 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:02,489 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:25:02,489 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 5489735189eabe07c0ddbb2d15b5812f, disabling compactions & flushes 2024-11-07T15:25:02,489 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:02,489 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:02,489 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. after waiting 0 ms 2024-11-07T15:25:02,489 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:02,492 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T15:25:02,493 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:02,493 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 5489735189eabe07c0ddbb2d15b5812f: Waiting for close lock at 1730993102489Running coprocessor pre-close hooks at 1730993102489Disabling compacts and flushes for region at 1730993102489Disabling writes for close at 1730993102489Writing region close event to WAL at 1730993102490 (+1 ms)Running coprocessor post-close hooks at 1730993102493 (+3 ms)Closed at 1730993102493 2024-11-07T15:25:02,493 WARN [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 5489735189eabe07c0ddbb2d15b5812f to self. 2024-11-07T15:25:02,494 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:02,494 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=5489735189eabe07c0ddbb2d15b5812f, regionState=CLOSED 2024-11-07T15:25:02,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:25:02,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-07T15:25:02,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 in 159 msec 2024-11-07T15:25:02,498 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5489735189eabe07c0ddbb2d15b5812f, REOPEN/MOVE; state=CLOSED, location=7b3e4e8e12ef,33849,1730992951906; forceNewPlan=false, retain=true 2024-11-07T15:25:02,648 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=5489735189eabe07c0ddbb2d15b5812f, regionState=OPENING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:02,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5489735189eabe07c0ddbb2d15b5812f, REOPEN/MOVE because future has completed 2024-11-07T15:25:02,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:25:02,804 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:02,804 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:25:02,804 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:02,804 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:25:02,804 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:02,804 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:02,805 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:02,806 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5489735189eabe07c0ddbb2d15b5812f columnFamilyName A 2024-11-07T15:25:02,807 DEBUG [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:02,807 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] regionserver.HStore(327): Store=5489735189eabe07c0ddbb2d15b5812f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:25:02,807 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:02,808 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5489735189eabe07c0ddbb2d15b5812f columnFamilyName B 2024-11-07T15:25:02,808 DEBUG [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:02,808 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] regionserver.HStore(327): Store=5489735189eabe07c0ddbb2d15b5812f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:25:02,808 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:02,808 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5489735189eabe07c0ddbb2d15b5812f columnFamilyName C 2024-11-07T15:25:02,808 DEBUG [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:02,809 INFO [StoreOpener-5489735189eabe07c0ddbb2d15b5812f-1 {}] regionserver.HStore(327): Store=5489735189eabe07c0ddbb2d15b5812f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:25:02,809 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:02,809 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:02,810 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:02,810 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:02,810 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:02,811 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:25:02,811 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:02,812 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 5489735189eabe07c0ddbb2d15b5812f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65163385, jitterRate=-0.028989896178245544}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:25:02,812 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:02,812 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 5489735189eabe07c0ddbb2d15b5812f: Running coprocessor pre-open hook at 1730993102804Writing region info on filesystem at 1730993102804Initializing all the Stores at 1730993102805 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993102805Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993102805Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730993102805Cleaning up temporary data from old regions at 1730993102810 (+5 ms)Running coprocessor post-open hooks at 1730993102812 (+2 ms)Region opened successfully at 1730993102812 2024-11-07T15:25:02,813 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., pid=151, masterSystemTime=1730993102802 2024-11-07T15:25:02,814 DEBUG [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:02,814 INFO [RS_OPEN_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:02,814 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=5489735189eabe07c0ddbb2d15b5812f, regionState=OPEN, openSeqNum=5, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:02,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:25:02,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-11-07T15:25:02,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 in 166 msec 2024-11-07T15:25:02,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-07T15:25:02,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5489735189eabe07c0ddbb2d15b5812f, REOPEN/MOVE in 482 msec 2024-11-07T15:25:02,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-07T15:25:02,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 486 msec 2024-11-07T15:25:02,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 900 msec 2024-11-07T15:25:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-07T15:25:02,822 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:25:02,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eb37c09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:25:02,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:25:02,824 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:25:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:25:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:25:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57d7626, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:25:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:25:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:02,825 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:25:02,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab87d6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d415d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:25:02,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:25:02,826 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:25:02,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:25:02,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:25:02,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@727d6a2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:25:02,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:25:02,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:02,827 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:25:02,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa63262, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e839bfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,828 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:25:02,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:25:02,829 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:25:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:25:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:25:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c508cf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:25:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:25:02,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:02,830 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:25:02,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a58e500, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31b8df26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,831 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:25:02,831 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:25:02,831 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:25:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:25:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:25:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3887ebe7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:25:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:25:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:02,832 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:25:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14252c34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f9db7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:25:02,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:25:02,834 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:25:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:25:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:25:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2099f738, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:25:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:25:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:02,835 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:25:02,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a472d06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e7631c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:25:02,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:25:02,836 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:25:02,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:25:02,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:25:02,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a6666ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:25:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:25:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:02,837 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:25:02,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5442f9e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35b8ab27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:25:02,838 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:25:02,839 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:25:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:25:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:25:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50287da1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:25:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:25:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:02,840 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:25:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@182c118c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e1056f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:25:02,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:25:02,842 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:25:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:25:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:25:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7535f476, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:25:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:25:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:02,843 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:25:02,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f850710, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@294719e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:25:02,844 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:25:02,845 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:25:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:25:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:25:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78dc1f91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:25:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:25:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:02,846 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:25:02,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4464d3f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@691c2b05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,847 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b3e4e8e12ef,46811,-1 for getting cluster id 2024-11-07T15:25:02,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:25:02,847 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '602c0594-20fc-4ac0-9c8a-0e1a48d3c68a' 2024-11-07T15:25:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:25:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "602c0594-20fc-4ac0-9c8a-0e1a48d3c68a" 2024-11-07T15:25:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@690b0445, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b3e4e8e12ef,46811,-1] 2024-11-07T15:25:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:25:02,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:02,848 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:25:02,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44fc2114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:25:02,849 DEBUG [Thread-2314 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:25:02,849 DEBUG [Thread-2309 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:25:02,849 DEBUG [Thread-2320 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:25:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:25:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:25:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:25:02,850 DEBUG [Thread-2303 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:25:02,850 DEBUG [Thread-2305 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:25:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:25:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:25:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:25:02,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:25:02,851 DEBUG [Thread-2307 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:25:02,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:25:02,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:25:02,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:25:02,851 DEBUG [Thread-2318 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:25:02,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:25:02,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:25:02,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:25:02,852 DEBUG [Thread-2307 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:25:02,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:25:02,852 DEBUG [Thread-2316 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:25:02,852 DEBUG [Thread-2307 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:25:02,852 DEBUG [Thread-2322 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:25:02,852 DEBUG [Thread-2311 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:25:02,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:25:02,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:25:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:25:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:25:02,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:25:02,853 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:25:02,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:25:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:25:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-07T15:25:02,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T15:25:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:25:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:25:02,855 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:25:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:25:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:25:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:25:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=-1] 2024-11-07T15:25:02,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:25:02,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:25:02,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:25:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:25:02,857 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:25:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:25:02,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:25:02,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:25:02,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:25:02,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:25:02,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:25:02,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:25:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:02,864 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:25:02,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:25:02,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5] 2024-11-07T15:25:02,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993162885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:02,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993162886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:02,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993162886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:02,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993162887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:02,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993162889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:02,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075323fe16c57c48a69516e89f5ae360ba_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993102862/Put/seqid=0 2024-11-07T15:25:02,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742340_1516 (size=12154) 2024-11-07T15:25:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T15:25:03,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993163001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:03,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993163001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:03,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993163001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:03,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:03,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993163001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:03,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:03,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993163001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:03,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:03,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T15:25:03,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:03,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:03,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:03,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T15:25:03,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:03,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:03,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:03,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T15:25:03,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993163211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:03,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993163211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993163211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993163211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993163211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:03,314 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:03,316 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075323fe16c57c48a69516e89f5ae360ba_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075323fe16c57c48a69516e89f5ae360ba_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:03,317 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/ea64d04f0cff42578c5ed9baacae4049, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:03,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/ea64d04f0cff42578c5ed9baacae4049 is 175, key is test_row_0/A:col10/1730993102862/Put/seqid=0 2024-11-07T15:25:03,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T15:25:03,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:03,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:03,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:03,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742341_1517 (size=30955) 2024-11-07T15:25:03,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T15:25:03,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:03,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:03,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:03,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T15:25:03,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993163521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:03,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993163521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:03,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993163521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993163521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:03,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993163521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:03,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:03,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T15:25:03,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:03,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:03,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:03,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,722 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/ea64d04f0cff42578c5ed9baacae4049 2024-11-07T15:25:03,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/563cf0201ce34071a3c85d1dbfa27033 is 50, key is test_row_0/B:col10/1730993102862/Put/seqid=0 2024-11-07T15:25:03,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742342_1518 (size=12001) 2024-11-07T15:25:03,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T15:25:03,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:03,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:03,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:03,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T15:25:03,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:03,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:03,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:03,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:03,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T15:25:04,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993164031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:04,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:04,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:04,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993164031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:04,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:04,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993164031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:04,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:04,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993164031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:04,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:04,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993164031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:04,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T15:25:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:04,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:04,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/563cf0201ce34071a3c85d1dbfa27033 2024-11-07T15:25:04,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/2a5f683b40d34cf688f5ca817c36012b is 50, key is test_row_0/C:col10/1730993102862/Put/seqid=0 2024-11-07T15:25:04,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742343_1519 (size=12001) 2024-11-07T15:25:04,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/2a5f683b40d34cf688f5ca817c36012b 2024-11-07T15:25:04,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/ea64d04f0cff42578c5ed9baacae4049 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/ea64d04f0cff42578c5ed9baacae4049 2024-11-07T15:25:04,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/ea64d04f0cff42578c5ed9baacae4049, entries=150, sequenceid=16, filesize=30.2 K 2024-11-07T15:25:04,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/563cf0201ce34071a3c85d1dbfa27033 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/563cf0201ce34071a3c85d1dbfa27033 2024-11-07T15:25:04,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/563cf0201ce34071a3c85d1dbfa27033, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T15:25:04,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/2a5f683b40d34cf688f5ca817c36012b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/2a5f683b40d34cf688f5ca817c36012b 2024-11-07T15:25:04,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/2a5f683b40d34cf688f5ca817c36012b, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T15:25:04,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5489735189eabe07c0ddbb2d15b5812f in 1345ms, sequenceid=16, compaction requested=false 2024-11-07T15:25:04,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:04,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T15:25:04,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:04,260 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T15:25:04,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072e89a0907b2945f987a25adf366746fd_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993102879/Put/seqid=0 2024-11-07T15:25:04,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742344_1520 (size=12154) 2024-11-07T15:25:04,281 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T15:25:04,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:04,674 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072e89a0907b2945f987a25adf366746fd_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072e89a0907b2945f987a25adf366746fd_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:04,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/23d126cc8ea44d7cb41cb005c8eeef36, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:04,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/23d126cc8ea44d7cb41cb005c8eeef36 is 175, key is test_row_0/A:col10/1730993102879/Put/seqid=0 2024-11-07T15:25:04,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742345_1521 (size=30955) 2024-11-07T15:25:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T15:25:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:05,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:05,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993165051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993165051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,055 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_0' in 5489735189eabe07c0ddbb2d15b5812f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:25:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993165053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993165054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993165055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,083 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/23d126cc8ea44d7cb41cb005c8eeef36 2024-11-07T15:25:05,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/03582e749d014b20ad2155802d07f592 is 50, key is test_row_0/B:col10/1730993102879/Put/seqid=0 2024-11-07T15:25:05,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742346_1522 (size=12001) 2024-11-07T15:25:05,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993165160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993165161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993165161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993165161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993165371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,374 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993165371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993165371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993165371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,510 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/03582e749d014b20ad2155802d07f592 2024-11-07T15:25:05,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/0f433ef42b434e649518b444a898035c is 50, key is test_row_0/C:col10/1730993102879/Put/seqid=0 2024-11-07T15:25:05,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742347_1523 (size=12001) 2024-11-07T15:25:05,519 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/0f433ef42b434e649518b444a898035c 2024-11-07T15:25:05,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/23d126cc8ea44d7cb41cb005c8eeef36 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/23d126cc8ea44d7cb41cb005c8eeef36 2024-11-07T15:25:05,525 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/23d126cc8ea44d7cb41cb005c8eeef36, entries=150, sequenceid=40, filesize=30.2 K 2024-11-07T15:25:05,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/03582e749d014b20ad2155802d07f592 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/03582e749d014b20ad2155802d07f592 2024-11-07T15:25:05,528 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/03582e749d014b20ad2155802d07f592, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T15:25:05,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/0f433ef42b434e649518b444a898035c as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/0f433ef42b434e649518b444a898035c 2024-11-07T15:25:05,533 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/0f433ef42b434e649518b444a898035c, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T15:25:05,533 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5489735189eabe07c0ddbb2d15b5812f in 1273ms, sequenceid=40, compaction requested=false 2024-11-07T15:25:05,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:05,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:05,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-07T15:25:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-07T15:25:05,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-07T15:25:05,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6770 sec 2024-11-07T15:25:05,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.6830 sec 2024-11-07T15:25:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:05,683 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:25:05,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T15:25:05,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079be77fea6a984aab80604239173c2f6f_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993105682/Put/seqid=0 2024-11-07T15:25:05,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742348_1524 (size=12154) 2024-11-07T15:25:05,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993165707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993165708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993165708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993165709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993165821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993165821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993165821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993165821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:05,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:05,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:06,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993166031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:06,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:06,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993166031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:06,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993166031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:06,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993166031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:06,096 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:06,101 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079be77fea6a984aab80604239173c2f6f_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079be77fea6a984aab80604239173c2f6f_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:06,102 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/882eba43c56146b1924a77093009aa76, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:06,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/882eba43c56146b1924a77093009aa76 is 175, key is test_row_0/A:col10/1730993105682/Put/seqid=0 2024-11-07T15:25:06,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742349_1525 (size=30955) 2024-11-07T15:25:06,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993166340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993166340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:06,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993166340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993166340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:06,520 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/882eba43c56146b1924a77093009aa76 2024-11-07T15:25:06,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/3b9de8068ec7488eab40c701696fca4f is 50, key is test_row_0/B:col10/1730993105682/Put/seqid=0 2024-11-07T15:25:06,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742350_1526 (size=12001) 2024-11-07T15:25:06,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/3b9de8068ec7488eab40c701696fca4f 2024-11-07T15:25:06,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/6f85c8dc164d4cf4a90ae16b926ec54b is 50, key is test_row_0/C:col10/1730993105682/Put/seqid=0 2024-11-07T15:25:06,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742351_1527 (size=12001) 2024-11-07T15:25:06,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993166850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:06,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993166850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993166850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:06,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993166850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:06,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/6f85c8dc164d4cf4a90ae16b926ec54b 2024-11-07T15:25:06,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/882eba43c56146b1924a77093009aa76 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/882eba43c56146b1924a77093009aa76 2024-11-07T15:25:06,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/882eba43c56146b1924a77093009aa76, entries=150, sequenceid=54, filesize=30.2 K 2024-11-07T15:25:06,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/3b9de8068ec7488eab40c701696fca4f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/3b9de8068ec7488eab40c701696fca4f 2024-11-07T15:25:06,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/3b9de8068ec7488eab40c701696fca4f, entries=150, sequenceid=54, filesize=11.7 K 2024-11-07T15:25:06,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/6f85c8dc164d4cf4a90ae16b926ec54b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/6f85c8dc164d4cf4a90ae16b926ec54b 2024-11-07T15:25:06,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/6f85c8dc164d4cf4a90ae16b926ec54b, entries=150, sequenceid=54, filesize=11.7 K 2024-11-07T15:25:06,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5489735189eabe07c0ddbb2d15b5812f in 1283ms, sequenceid=54, compaction requested=true 2024-11-07T15:25:06,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:06,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:25:06,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:06,966 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:06,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:25:06,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:06,966 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:06,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:25:06,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:06,967 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:06,967 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/B is initiating minor compaction (all files) 2024-11-07T15:25:06,967 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/B in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:06,967 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:06,967 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/A is initiating minor compaction (all files) 2024-11-07T15:25:06,967 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/563cf0201ce34071a3c85d1dbfa27033, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/03582e749d014b20ad2155802d07f592, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/3b9de8068ec7488eab40c701696fca4f] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=35.2 K 2024-11-07T15:25:06,967 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/A in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:06,967 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/ea64d04f0cff42578c5ed9baacae4049, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/23d126cc8ea44d7cb41cb005c8eeef36, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/882eba43c56146b1924a77093009aa76] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=90.7 K 2024-11-07T15:25:06,967 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:06,968 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/ea64d04f0cff42578c5ed9baacae4049, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/23d126cc8ea44d7cb41cb005c8eeef36, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/882eba43c56146b1924a77093009aa76] 2024-11-07T15:25:06,968 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea64d04f0cff42578c5ed9baacae4049, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730993102862 2024-11-07T15:25:06,968 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 563cf0201ce34071a3c85d1dbfa27033, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730993102862 2024-11-07T15:25:06,968 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23d126cc8ea44d7cb41cb005c8eeef36, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730993102879 2024-11-07T15:25:06,968 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 03582e749d014b20ad2155802d07f592, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730993102879 2024-11-07T15:25:06,968 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b9de8068ec7488eab40c701696fca4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730993105053 2024-11-07T15:25:06,968 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 882eba43c56146b1924a77093009aa76, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730993105053 2024-11-07T15:25:06,975 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#B#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:06,975 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/790e28c55eaa41f58b726b03b436ba1e is 50, key is test_row_0/B:col10/1730993105682/Put/seqid=0 2024-11-07T15:25:06,976 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:06,979 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411078880d8639a9546d2bea968cbe5fb6a67_5489735189eabe07c0ddbb2d15b5812f store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:06,981 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411078880d8639a9546d2bea968cbe5fb6a67_5489735189eabe07c0ddbb2d15b5812f, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:06,981 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078880d8639a9546d2bea968cbe5fb6a67_5489735189eabe07c0ddbb2d15b5812f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:06,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742352_1528 (size=12104) 2024-11-07T15:25:06,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742353_1529 (size=4469) 2024-11-07T15:25:06,995 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#A#compaction#446 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:06,995 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/d9767f8f2b2f47d6be2cc4a1c34e6e9b is 175, key is test_row_0/A:col10/1730993105682/Put/seqid=0 2024-11-07T15:25:06,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742354_1530 (size=31058) 2024-11-07T15:25:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T15:25:07,031 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:25:07,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:25:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-07T15:25:07,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T15:25:07,032 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:25:07,033 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:25:07,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:25:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:07,073 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:25:07,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T15:25:07,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ca825e986a5c46138150b79214c5263f_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993107073/Put/seqid=0 2024-11-07T15:25:07,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742355_1531 (size=14594) 2024-11-07T15:25:07,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:07,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993167117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:07,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T15:25:07,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-07T15:25:07,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:07,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:07,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:07,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:07,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:07,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993167230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:07,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-07T15:25:07,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:07,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:07,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:07,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:07,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T15:25:07,386 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/790e28c55eaa41f58b726b03b436ba1e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/790e28c55eaa41f58b726b03b436ba1e 2024-11-07T15:25:07,389 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/B of 5489735189eabe07c0ddbb2d15b5812f into 790e28c55eaa41f58b726b03b436ba1e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:07,389 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:07,389 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/B, priority=13, startTime=1730993106966; duration=0sec 2024-11-07T15:25:07,389 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:07,389 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:B 2024-11-07T15:25:07,389 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:07,390 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:07,390 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/C is initiating minor compaction (all files) 2024-11-07T15:25:07,390 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/C in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:07,390 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/2a5f683b40d34cf688f5ca817c36012b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/0f433ef42b434e649518b444a898035c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/6f85c8dc164d4cf4a90ae16b926ec54b] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=35.2 K 2024-11-07T15:25:07,390 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a5f683b40d34cf688f5ca817c36012b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730993102862 2024-11-07T15:25:07,391 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f433ef42b434e649518b444a898035c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730993102879 2024-11-07T15:25:07,391 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f85c8dc164d4cf4a90ae16b926ec54b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730993105053 2024-11-07T15:25:07,400 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#C#compaction#448 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:07,401 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/4b60647ce54243bf88d5bd77d9f90613 is 50, key is test_row_0/C:col10/1730993105682/Put/seqid=0 2024-11-07T15:25:07,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742356_1532 (size=12104) 2024-11-07T15:25:07,409 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/d9767f8f2b2f47d6be2cc4a1c34e6e9b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/d9767f8f2b2f47d6be2cc4a1c34e6e9b 2024-11-07T15:25:07,412 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/4b60647ce54243bf88d5bd77d9f90613 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/4b60647ce54243bf88d5bd77d9f90613 2024-11-07T15:25:07,414 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/A of 5489735189eabe07c0ddbb2d15b5812f into d9767f8f2b2f47d6be2cc4a1c34e6e9b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:07,414 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:07,414 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/A, priority=13, startTime=1730993106966; duration=0sec 2024-11-07T15:25:07,414 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:07,414 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:A 2024-11-07T15:25:07,418 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/C of 5489735189eabe07c0ddbb2d15b5812f into 4b60647ce54243bf88d5bd77d9f90613(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:07,418 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:07,418 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/C, priority=13, startTime=1730993106966; duration=0sec 2024-11-07T15:25:07,418 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:07,418 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:C 2024-11-07T15:25:07,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993167441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:07,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:07,485 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:07,487 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ca825e986a5c46138150b79214c5263f_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ca825e986a5c46138150b79214c5263f_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:07,488 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/f2fd0a83b4cf4191ac641b64db2b4f54, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:07,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/f2fd0a83b4cf4191ac641b64db2b4f54 is 175, key is test_row_0/A:col10/1730993107073/Put/seqid=0 2024-11-07T15:25:07,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-07T15:25:07,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:07,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:07,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:07,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:07,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:07,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742357_1533 (size=39549) 2024-11-07T15:25:07,492 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/f2fd0a83b4cf4191ac641b64db2b4f54 2024-11-07T15:25:07,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/dc7ce05bf9784a87851c18ba0aef1ad3 is 50, key is test_row_0/B:col10/1730993107073/Put/seqid=0 2024-11-07T15:25:07,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742358_1534 (size=12001) 2024-11-07T15:25:07,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/dc7ce05bf9784a87851c18ba0aef1ad3 2024-11-07T15:25:07,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/755868d5f90c44629900330b2f6e3635 is 50, key is test_row_0/C:col10/1730993107073/Put/seqid=0 2024-11-07T15:25:07,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742359_1535 (size=12001) 2024-11-07T15:25:07,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/755868d5f90c44629900330b2f6e3635 2024-11-07T15:25:07,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/f2fd0a83b4cf4191ac641b64db2b4f54 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/f2fd0a83b4cf4191ac641b64db2b4f54 2024-11-07T15:25:07,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/f2fd0a83b4cf4191ac641b64db2b4f54, entries=200, sequenceid=77, filesize=38.6 K 2024-11-07T15:25:07,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/dc7ce05bf9784a87851c18ba0aef1ad3 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/dc7ce05bf9784a87851c18ba0aef1ad3 2024-11-07T15:25:07,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/dc7ce05bf9784a87851c18ba0aef1ad3, entries=150, sequenceid=77, filesize=11.7 K 2024-11-07T15:25:07,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/755868d5f90c44629900330b2f6e3635 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/755868d5f90c44629900330b2f6e3635 2024-11-07T15:25:07,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/755868d5f90c44629900330b2f6e3635, entries=150, sequenceid=77, filesize=11.7 K 2024-11-07T15:25:07,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 5489735189eabe07c0ddbb2d15b5812f in 449ms, sequenceid=77, compaction requested=false 2024-11-07T15:25:07,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:07,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-07T15:25:07,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:07,643 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T15:25:07,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107803604eb22034918a9f5b6dc9aaf859c_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993107106/Put/seqid=0 2024-11-07T15:25:07,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742360_1536 (size=12154) 2024-11-07T15:25:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T15:25:07,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:07,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:07,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993167828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:07,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993167860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:07,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993167860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:07,862 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in 5489735189eabe07c0ddbb2d15b5812f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:25:07,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:07,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993167860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:07,863 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_1' in 5489735189eabe07c0ddbb2d15b5812f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:25:07,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:07,864 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_1' in 5489735189eabe07c0ddbb2d15b5812f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:25:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:07,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:07,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993167871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:07,873 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_1' in 5489735189eabe07c0ddbb2d15b5812f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:25:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:07,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:07,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993167940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:07,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:08,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:08,062 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107803604eb22034918a9f5b6dc9aaf859c_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107803604eb22034918a9f5b6dc9aaf859c_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:08,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/636160b5f6b548b88561ae2ee28f66d6, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:08,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/636160b5f6b548b88561ae2ee28f66d6 is 175, key is test_row_0/A:col10/1730993107106/Put/seqid=0 2024-11-07T15:25:08,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742361_1537 (size=30955) 2024-11-07T15:25:08,072 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/636160b5f6b548b88561ae2ee28f66d6 2024-11-07T15:25:08,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/75825efcbb844b7986dff42a064d5522 is 50, key is test_row_0/B:col10/1730993107106/Put/seqid=0 2024-11-07T15:25:08,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742362_1538 (size=12001) 2024-11-07T15:25:08,081 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/75825efcbb844b7986dff42a064d5522 2024-11-07T15:25:08,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/e3527c7ab45d435a9ed3ffe64b379710 is 50, key is test_row_0/C:col10/1730993107106/Put/seqid=0 2024-11-07T15:25:08,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742363_1539 (size=12001) 2024-11-07T15:25:08,093 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/e3527c7ab45d435a9ed3ffe64b379710 2024-11-07T15:25:08,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/636160b5f6b548b88561ae2ee28f66d6 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/636160b5f6b548b88561ae2ee28f66d6 2024-11-07T15:25:08,100 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/636160b5f6b548b88561ae2ee28f66d6, entries=150, sequenceid=93, filesize=30.2 K 2024-11-07T15:25:08,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/75825efcbb844b7986dff42a064d5522 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/75825efcbb844b7986dff42a064d5522 2024-11-07T15:25:08,104 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/75825efcbb844b7986dff42a064d5522, entries=150, sequenceid=93, filesize=11.7 K 2024-11-07T15:25:08,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/e3527c7ab45d435a9ed3ffe64b379710 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e3527c7ab45d435a9ed3ffe64b379710 2024-11-07T15:25:08,107 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e3527c7ab45d435a9ed3ffe64b379710, entries=150, sequenceid=93, filesize=11.7 K 2024-11-07T15:25:08,108 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5489735189eabe07c0ddbb2d15b5812f in 466ms, sequenceid=93, compaction requested=true 2024-11-07T15:25:08,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:08,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:08,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-07T15:25:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-07T15:25:08,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-07T15:25:08,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0760 sec 2024-11-07T15:25:08,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.0800 sec 2024-11-07T15:25:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:08,154 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:25:08,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T15:25:08,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107707563a2f93a4aadb46bb5366270f600_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993108153/Put/seqid=0 2024-11-07T15:25:08,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742364_1540 (size=14594) 2024-11-07T15:25:08,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T15:25:08,170 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:25:08,171 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:25:08,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-07T15:25:08,172 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:25:08,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T15:25:08,173 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:25:08,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:25:08,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993168210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:08,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T15:25:08,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993168320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:08,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T15:25:08,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:08,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:08,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:08,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:08,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:08,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T15:25:08,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:08,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:08,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:08,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:08,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T15:25:08,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:08,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993168531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:08,562 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:08,565 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107707563a2f93a4aadb46bb5366270f600_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107707563a2f93a4aadb46bb5366270f600_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:08,566 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/a88c44cbfaf040cda14ea331f1271c53, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:08,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/a88c44cbfaf040cda14ea331f1271c53 is 175, key is test_row_0/A:col10/1730993108153/Put/seqid=0 2024-11-07T15:25:08,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742365_1541 (size=39549) 2024-11-07T15:25:08,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T15:25:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:08,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:08,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:08,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:08,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T15:25:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:08,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:08,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:08,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T15:25:08,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993168840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:08,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:08,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:08,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:08,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T15:25:08,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:08,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:08,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:08,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:08,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:08,994 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/a88c44cbfaf040cda14ea331f1271c53 2024-11-07T15:25:09,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/f8c85e587aee471289d6573077192738 is 50, key is test_row_0/B:col10/1730993108153/Put/seqid=0 2024-11-07T15:25:09,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742366_1542 (size=12001) 2024-11-07T15:25:09,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/f8c85e587aee471289d6573077192738 2024-11-07T15:25:09,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/1113ddd2b7bf47b2bc7c381cc98fae5c is 50, key is test_row_0/C:col10/1730993108153/Put/seqid=0 2024-11-07T15:25:09,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742367_1543 (size=12001) 2024-11-07T15:25:09,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T15:25:09,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:09,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:09,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:09,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:09,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:09,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T15:25:09,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:09,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:09,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:09,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:09,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T15:25:09,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993169350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:09,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/1113ddd2b7bf47b2bc7c381cc98fae5c 2024-11-07T15:25:09,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/a88c44cbfaf040cda14ea331f1271c53 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/a88c44cbfaf040cda14ea331f1271c53 2024-11-07T15:25:09,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/a88c44cbfaf040cda14ea331f1271c53, entries=200, sequenceid=117, filesize=38.6 K 2024-11-07T15:25:09,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/f8c85e587aee471289d6573077192738 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/f8c85e587aee471289d6573077192738 2024-11-07T15:25:09,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/f8c85e587aee471289d6573077192738, entries=150, sequenceid=117, filesize=11.7 K 2024-11-07T15:25:09,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/1113ddd2b7bf47b2bc7c381cc98fae5c as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1113ddd2b7bf47b2bc7c381cc98fae5c 2024-11-07T15:25:09,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1113ddd2b7bf47b2bc7c381cc98fae5c, entries=150, sequenceid=117, filesize=11.7 K 2024-11-07T15:25:09,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5489735189eabe07c0ddbb2d15b5812f in 1273ms, sequenceid=117, compaction requested=true 2024-11-07T15:25:09,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:09,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:25:09,427 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:25:09,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:09,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:25:09,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:09,427 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:25:09,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:25:09,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:09,428 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:25:09,428 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/A is initiating minor compaction (all files) 2024-11-07T15:25:09,428 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/A in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:09,428 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/d9767f8f2b2f47d6be2cc4a1c34e6e9b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/f2fd0a83b4cf4191ac641b64db2b4f54, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/636160b5f6b548b88561ae2ee28f66d6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/a88c44cbfaf040cda14ea331f1271c53] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=137.8 K 2024-11-07T15:25:09,428 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:09,428 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/d9767f8f2b2f47d6be2cc4a1c34e6e9b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/f2fd0a83b4cf4191ac641b64db2b4f54, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/636160b5f6b548b88561ae2ee28f66d6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/a88c44cbfaf040cda14ea331f1271c53] 2024-11-07T15:25:09,429 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9767f8f2b2f47d6be2cc4a1c34e6e9b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730993105053 2024-11-07T15:25:09,429 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:25:09,429 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/B is initiating minor compaction (all files) 2024-11-07T15:25:09,429 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/B in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:09,429 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/790e28c55eaa41f58b726b03b436ba1e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/dc7ce05bf9784a87851c18ba0aef1ad3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/75825efcbb844b7986dff42a064d5522, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/f8c85e587aee471289d6573077192738] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=47.0 K 2024-11-07T15:25:09,430 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2fd0a83b4cf4191ac641b64db2b4f54, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730993105707 2024-11-07T15:25:09,430 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 790e28c55eaa41f58b726b03b436ba1e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730993105053 2024-11-07T15:25:09,430 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting dc7ce05bf9784a87851c18ba0aef1ad3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730993105707 2024-11-07T15:25:09,430 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 636160b5f6b548b88561ae2ee28f66d6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730993107106 2024-11-07T15:25:09,430 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 75825efcbb844b7986dff42a064d5522, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730993107106 2024-11-07T15:25:09,430 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting a88c44cbfaf040cda14ea331f1271c53, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730993107810 2024-11-07T15:25:09,431 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting f8c85e587aee471289d6573077192738, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730993107814 2024-11-07T15:25:09,438 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#B#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:09,438 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/4b3d31f844f4479c96ba0ec39efd66a1 is 50, key is test_row_0/B:col10/1730993108153/Put/seqid=0 2024-11-07T15:25:09,439 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:09,441 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107be0e8f3de80d49ba9e63f9b1eb829e8c_5489735189eabe07c0ddbb2d15b5812f store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:09,443 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107be0e8f3de80d49ba9e63f9b1eb829e8c_5489735189eabe07c0ddbb2d15b5812f, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:09,443 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107be0e8f3de80d49ba9e63f9b1eb829e8c_5489735189eabe07c0ddbb2d15b5812f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:09,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742368_1544 (size=12241) 2024-11-07T15:25:09,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742369_1545 (size=4469) 2024-11-07T15:25:09,447 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#A#compaction#458 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:09,447 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/f7b9695efc9b42368c34532845067faf is 175, key is test_row_0/A:col10/1730993108153/Put/seqid=0 2024-11-07T15:25:09,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T15:25:09,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:09,450 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T15:25:09,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742370_1546 (size=31195) 2024-11-07T15:25:09,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070acae8bd65a446798c97fedd01cea9d6_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993108190/Put/seqid=0 2024-11-07T15:25:09,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742371_1547 (size=12154) 2024-11-07T15:25:09,848 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/4b3d31f844f4479c96ba0ec39efd66a1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/4b3d31f844f4479c96ba0ec39efd66a1 2024-11-07T15:25:09,851 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/B of 5489735189eabe07c0ddbb2d15b5812f into 4b3d31f844f4479c96ba0ec39efd66a1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:09,851 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:09,851 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/B, priority=12, startTime=1730993109427; duration=0sec 2024-11-07T15:25:09,851 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:09,851 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:B 2024-11-07T15:25:09,851 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:25:09,853 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:25:09,853 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/C is initiating minor compaction (all files) 2024-11-07T15:25:09,853 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/C in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:09,853 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/4b60647ce54243bf88d5bd77d9f90613, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/755868d5f90c44629900330b2f6e3635, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e3527c7ab45d435a9ed3ffe64b379710, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1113ddd2b7bf47b2bc7c381cc98fae5c] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=47.0 K 2024-11-07T15:25:09,854 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b60647ce54243bf88d5bd77d9f90613, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1730993105053 2024-11-07T15:25:09,854 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 755868d5f90c44629900330b2f6e3635, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730993105707 2024-11-07T15:25:09,855 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting e3527c7ab45d435a9ed3ffe64b379710, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1730993107106 2024-11-07T15:25:09,856 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/f7b9695efc9b42368c34532845067faf as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/f7b9695efc9b42368c34532845067faf 2024-11-07T15:25:09,856 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 1113ddd2b7bf47b2bc7c381cc98fae5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730993107814 2024-11-07T15:25:09,860 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/A of 5489735189eabe07c0ddbb2d15b5812f into f7b9695efc9b42368c34532845067faf(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:09,860 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:09,860 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/A, priority=12, startTime=1730993109427; duration=0sec 2024-11-07T15:25:09,860 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:09,860 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:A 2024-11-07T15:25:09,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:09,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:09,876 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070acae8bd65a446798c97fedd01cea9d6_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070acae8bd65a446798c97fedd01cea9d6_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:09,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/8dde1d0b8d244968adb38e3c12332eba, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:09,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/8dde1d0b8d244968adb38e3c12332eba is 175, key is test_row_0/A:col10/1730993108190/Put/seqid=0 2024-11-07T15:25:09,890 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#C#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:09,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742372_1548 (size=30955) 2024-11-07T15:25:09,891 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/e95222c107f648989ddc1d0b58f67e36 is 50, key is test_row_0/C:col10/1730993108153/Put/seqid=0 2024-11-07T15:25:09,891 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/8dde1d0b8d244968adb38e3c12332eba 2024-11-07T15:25:09,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742373_1549 (size=12241) 2024-11-07T15:25:09,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/75d4dd459e734c5ea83efb4b4c495fea is 50, key is test_row_0/B:col10/1730993108190/Put/seqid=0 2024-11-07T15:25:09,904 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/e95222c107f648989ddc1d0b58f67e36 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e95222c107f648989ddc1d0b58f67e36 2024-11-07T15:25:09,908 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/C of 5489735189eabe07c0ddbb2d15b5812f into e95222c107f648989ddc1d0b58f67e36(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:09,908 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:09,908 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/C, priority=12, startTime=1730993109427; duration=0sec 2024-11-07T15:25:09,908 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:09,908 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:C 2024-11-07T15:25:09,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742374_1550 (size=12001) 2024-11-07T15:25:09,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993169904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:09,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993169905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:09,910 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/75d4dd459e734c5ea83efb4b4c495fea 2024-11-07T15:25:09,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:09,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993169908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993169908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:09,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/385397b46fc04a429457906aea48f355 is 50, key is test_row_0/C:col10/1730993108190/Put/seqid=0 2024-11-07T15:25:09,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742375_1551 (size=12001) 2024-11-07T15:25:09,922 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/385397b46fc04a429457906aea48f355 2024-11-07T15:25:09,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/8dde1d0b8d244968adb38e3c12332eba as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/8dde1d0b8d244968adb38e3c12332eba 2024-11-07T15:25:09,930 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/8dde1d0b8d244968adb38e3c12332eba, entries=150, sequenceid=129, filesize=30.2 K 2024-11-07T15:25:09,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/75d4dd459e734c5ea83efb4b4c495fea as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/75d4dd459e734c5ea83efb4b4c495fea 2024-11-07T15:25:09,933 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/75d4dd459e734c5ea83efb4b4c495fea, entries=150, sequenceid=129, filesize=11.7 K 2024-11-07T15:25:09,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/385397b46fc04a429457906aea48f355 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/385397b46fc04a429457906aea48f355 2024-11-07T15:25:09,937 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/385397b46fc04a429457906aea48f355, entries=150, sequenceid=129, filesize=11.7 K 2024-11-07T15:25:09,937 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 5489735189eabe07c0ddbb2d15b5812f in 487ms, sequenceid=129, compaction requested=false 2024-11-07T15:25:09,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:09,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:09,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-07T15:25:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-07T15:25:09,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-07T15:25:09,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7650 sec 2024-11-07T15:25:09,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.7740 sec 2024-11-07T15:25:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:10,024 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:25:10,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T15:25:10,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b7239d82b6a6486ebeb9b88f6dc3f896_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993110023/Put/seqid=0 2024-11-07T15:25:10,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742376_1552 (size=17284) 2024-11-07T15:25:10,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993170030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993170031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993170031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993170031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993170141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993170141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993170141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993170141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T15:25:10,321 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:25:10,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:25:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-07T15:25:10,322 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:25:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T15:25:10,323 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:25:10,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:25:10,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993170351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993170351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993170351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993170352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993170370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,372 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in 5489735189eabe07c0ddbb2d15b5812f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:25:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T15:25:10,434 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:10,437 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b7239d82b6a6486ebeb9b88f6dc3f896_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b7239d82b6a6486ebeb9b88f6dc3f896_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:10,437 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/b2f36fc5e838445aab180e6c2e558cae, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:10,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/b2f36fc5e838445aab180e6c2e558cae is 175, key is test_row_0/A:col10/1730993110023/Put/seqid=0 2024-11-07T15:25:10,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742377_1553 (size=48389) 2024-11-07T15:25:10,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T15:25:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:10,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:10,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T15:25:10,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:10,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:10,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:10,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:10,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:10,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T15:25:10,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993170661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993170661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993170661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993170661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:10,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T15:25:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:10,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:10,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:10,841 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/b2f36fc5e838445aab180e6c2e558cae 2024-11-07T15:25:10,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/7826bbaea98745d9b77e6b0f014c6e38 is 50, key is test_row_0/B:col10/1730993110023/Put/seqid=0 2024-11-07T15:25:10,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742378_1554 (size=12151) 2024-11-07T15:25:10,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/7826bbaea98745d9b77e6b0f014c6e38 2024-11-07T15:25:10,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/1d8b1abdbf30481cbfd5903bd0b200a4 is 50, key is test_row_0/C:col10/1730993110023/Put/seqid=0 2024-11-07T15:25:10,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742379_1555 (size=12151) 2024-11-07T15:25:10,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T15:25:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:10,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:10,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T15:25:11,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T15:25:11,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:11,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:11,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:11,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:11,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:11,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993171171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:11,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993171171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:11,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993171171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:11,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993171171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:11,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T15:25:11,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:11,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:11,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:11,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:11,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:11,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/1d8b1abdbf30481cbfd5903bd0b200a4 2024-11-07T15:25:11,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/b2f36fc5e838445aab180e6c2e558cae as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/b2f36fc5e838445aab180e6c2e558cae 2024-11-07T15:25:11,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/b2f36fc5e838445aab180e6c2e558cae, entries=250, sequenceid=159, filesize=47.3 K 2024-11-07T15:25:11,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/7826bbaea98745d9b77e6b0f014c6e38 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/7826bbaea98745d9b77e6b0f014c6e38 2024-11-07T15:25:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/7826bbaea98745d9b77e6b0f014c6e38, entries=150, sequenceid=159, filesize=11.9 K 2024-11-07T15:25:11,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/1d8b1abdbf30481cbfd5903bd0b200a4 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1d8b1abdbf30481cbfd5903bd0b200a4 2024-11-07T15:25:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1d8b1abdbf30481cbfd5903bd0b200a4, entries=150, sequenceid=159, filesize=11.9 K 2024-11-07T15:25:11,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 5489735189eabe07c0ddbb2d15b5812f in 1251ms, sequenceid=159, compaction requested=true 2024-11-07T15:25:11,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:11,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:25:11,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:11,276 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:11,276 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:11,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:25:11,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:11,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:25:11,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:11,276 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110539 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:11,276 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:11,276 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/A is initiating minor compaction (all files) 2024-11-07T15:25:11,276 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/B is initiating minor compaction (all files) 2024-11-07T15:25:11,276 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/B in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:11,276 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/A in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:11,277 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/f7b9695efc9b42368c34532845067faf, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/8dde1d0b8d244968adb38e3c12332eba, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/b2f36fc5e838445aab180e6c2e558cae] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=107.9 K 2024-11-07T15:25:11,277 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/4b3d31f844f4479c96ba0ec39efd66a1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/75d4dd459e734c5ea83efb4b4c495fea, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/7826bbaea98745d9b77e6b0f014c6e38] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=35.5 K 2024-11-07T15:25:11,277 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:11,277 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/f7b9695efc9b42368c34532845067faf, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/8dde1d0b8d244968adb38e3c12332eba, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/b2f36fc5e838445aab180e6c2e558cae] 2024-11-07T15:25:11,277 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b3d31f844f4479c96ba0ec39efd66a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730993107814 2024-11-07T15:25:11,277 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7b9695efc9b42368c34532845067faf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730993107814 2024-11-07T15:25:11,277 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 75d4dd459e734c5ea83efb4b4c495fea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1730993108177 2024-11-07T15:25:11,277 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dde1d0b8d244968adb38e3c12332eba, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1730993108177 2024-11-07T15:25:11,277 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 7826bbaea98745d9b77e6b0f014c6e38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730993109906 2024-11-07T15:25:11,277 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2f36fc5e838445aab180e6c2e558cae, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730993109902 2024-11-07T15:25:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,282 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,283 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#B#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,283 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/9b3b99e44def4779a77e45255851f661 is 50, key is test_row_0/B:col10/1730993110023/Put/seqid=0 2024-11-07T15:25:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,284 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110747069c0b2c2b41b49195629ee145cb84_5489735189eabe07c0ddbb2d15b5812f store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,285 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110747069c0b2c2b41b49195629ee145cb84_5489735189eabe07c0ddbb2d15b5812f, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,286 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110747069c0b2c2b41b49195629ee145cb84_5489735189eabe07c0ddbb2d15b5812f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742381_1557 (size=4469) 2024-11-07T15:25:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742380_1556 (size=12493) 2024-11-07T15:25:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,297 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#A#compaction#467 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,297 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/fcd5a51d02ce47ab92016cff28499c2b is 175, key is test_row_0/A:col10/1730993110023/Put/seqid=0 2024-11-07T15:25:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742382_1558 (size=31447) 2024-11-07T15:25:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T15:25:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:11,392 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-07T15:25:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bc0fcce2f4c540548afb6b0f2a4f9ffe_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_1/A:col10/1730993110028/Put/seqid=0 2024-11-07T15:25:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742383_1559 (size=9814) 2024-11-07T15:25:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T15:25:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,696 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/9b3b99e44def4779a77e45255851f661 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/9b3b99e44def4779a77e45255851f661 2024-11-07T15:25:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,701 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/B of 5489735189eabe07c0ddbb2d15b5812f into 9b3b99e44def4779a77e45255851f661(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:11,701 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:11,701 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/B, priority=13, startTime=1730993111276; duration=0sec 2024-11-07T15:25:11,701 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:11,701 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:B 2024-11-07T15:25:11,701 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,702 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:11,702 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/C is initiating minor compaction (all files) 2024-11-07T15:25:11,702 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/C in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:11,702 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e95222c107f648989ddc1d0b58f67e36, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/385397b46fc04a429457906aea48f355, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1d8b1abdbf30481cbfd5903bd0b200a4] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=35.5 K 2024-11-07T15:25:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,703 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting e95222c107f648989ddc1d0b58f67e36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730993107814 2024-11-07T15:25:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,703 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 385397b46fc04a429457906aea48f355, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1730993108177 2024-11-07T15:25:11,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,704 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d8b1abdbf30481cbfd5903bd0b200a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730993109906 2024-11-07T15:25:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,705 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/fcd5a51d02ce47ab92016cff28499c2b as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/fcd5a51d02ce47ab92016cff28499c2b 2024-11-07T15:25:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,709 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/A of 5489735189eabe07c0ddbb2d15b5812f into fcd5a51d02ce47ab92016cff28499c2b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:11,709 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,709 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/A, priority=13, startTime=1730993111275; duration=0sec 2024-11-07T15:25:11,709 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:11,709 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:A 2024-11-07T15:25:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,713 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#C#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,713 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/aef9f8bebb704b92959d429bfdd9d498 is 50, key is test_row_0/C:col10/1730993110023/Put/seqid=0 2024-11-07T15:25:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742384_1560 (size=12493) 2024-11-07T15:25:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,734 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/aef9f8bebb704b92959d429bfdd9d498 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/aef9f8bebb704b92959d429bfdd9d498 2024-11-07T15:25:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,738 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/C of 5489735189eabe07c0ddbb2d15b5812f into aef9f8bebb704b92959d429bfdd9d498(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:11,738 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:11,738 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/C, priority=13, startTime=1730993111276; duration=0sec 2024-11-07T15:25:11,738 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:11,738 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:C 2024-11-07T15:25:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,803 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bc0fcce2f4c540548afb6b0f2a4f9ffe_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bc0fcce2f4c540548afb6b0f2a4f9ffe_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/3f4f26ca83794876a74f36884ca93468, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/3f4f26ca83794876a74f36884ca93468 is 175, key is test_row_1/A:col10/1730993110028/Put/seqid=0 2024-11-07T15:25:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742385_1561 (size=22461) 2024-11-07T15:25:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,817 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/3f4f26ca83794876a74f36884ca93468 2024-11-07T15:25:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/bce0a27df8a34f2f9340aa963bc21a38 is 50, key is test_row_1/B:col10/1730993110028/Put/seqid=0 2024-11-07T15:25:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742386_1562 (size=9757) 2024-11-07T15:25:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,831 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/bce0a27df8a34f2f9340aa963bc21a38 2024-11-07T15:25:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/24766d8c28c24fd2913ee5f31752192c is 50, key is test_row_1/C:col10/1730993110028/Put/seqid=0 2024-11-07T15:25:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742387_1563 (size=9757) 2024-11-07T15:25:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,854 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/24766d8c28c24fd2913ee5f31752192c 2024-11-07T15:25:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/3f4f26ca83794876a74f36884ca93468 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/3f4f26ca83794876a74f36884ca93468 2024-11-07T15:25:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,863 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/3f4f26ca83794876a74f36884ca93468, entries=100, sequenceid=168, filesize=21.9 K 2024-11-07T15:25:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/bce0a27df8a34f2f9340aa963bc21a38 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/bce0a27df8a34f2f9340aa963bc21a38 2024-11-07T15:25:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,867 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/bce0a27df8a34f2f9340aa963bc21a38, entries=100, sequenceid=168, filesize=9.5 K 2024-11-07T15:25:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/24766d8c28c24fd2913ee5f31752192c as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/24766d8c28c24fd2913ee5f31752192c 2024-11-07T15:25:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,871 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/24766d8c28c24fd2913ee5f31752192c, entries=100, sequenceid=168, filesize=9.5 K 2024-11-07T15:25:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,872 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 5489735189eabe07c0ddbb2d15b5812f in 480ms, sequenceid=168, compaction requested=false 2024-11-07T15:25:11,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:11,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:11,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-07T15:25:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-07T15:25:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-07T15:25:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5500 sec 2024-11-07T15:25:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.5550 sec 2024-11-07T15:25:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,219 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:25:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:25:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a8828c673ae040a6a5b50cb87af11c1e_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993112218/Put/seqid=0 2024-11-07T15:25:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742389_1565 (size=24758) 2024-11-07T15:25:12,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993172269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993172271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993172273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993172274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993172381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993172381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993172381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993172381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993172390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,393 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in 5489735189eabe07c0ddbb2d15b5812f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:25:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T15:25:12,471 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:25:12,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:25:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-07T15:25:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T15:25:12,472 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:25:12,473 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:25:12,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:25:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T15:25:12,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993172590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993172590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993172590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993172591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T15:25:12,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:12,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:12,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:12,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:12,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:12,666 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:12,669 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a8828c673ae040a6a5b50cb87af11c1e_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a8828c673ae040a6a5b50cb87af11c1e_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:12,670 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/4e9acc68bd9d45ed95a9e8faf3cbca78, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:12,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/4e9acc68bd9d45ed95a9e8faf3cbca78 is 175, key is test_row_0/A:col10/1730993112218/Put/seqid=0 2024-11-07T15:25:12,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742388_1564 (size=74391) 2024-11-07T15:25:12,672 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/4e9acc68bd9d45ed95a9e8faf3cbca78 2024-11-07T15:25:12,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/874f9af43b394fdea8dac63ac82077d8 is 50, key is test_row_0/B:col10/1730993112218/Put/seqid=0 2024-11-07T15:25:12,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742390_1566 (size=9757) 2024-11-07T15:25:12,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T15:25:12,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:12,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:12,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:12,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:12,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T15:25:12,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993172901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993172901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993172901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993172901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:12,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T15:25:12,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:12,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:12,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:12,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:12,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:12,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:13,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T15:25:13,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:13,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:13,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:13,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:13,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:13,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/874f9af43b394fdea8dac63ac82077d8 2024-11-07T15:25:13,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/c9b3c2fd7a45403c887198932b308e8d is 50, key is test_row_0/C:col10/1730993112218/Put/seqid=0 2024-11-07T15:25:13,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742391_1567 (size=9757) 2024-11-07T15:25:13,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T15:25:13,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T15:25:13,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:13,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:13,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:13,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:13,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:13,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:13,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T15:25:13,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:13,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:13,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:13,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:13,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:13,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993173411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:13,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:13,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:13,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993173411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:13,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993173411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:13,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993173411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:13,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/c9b3c2fd7a45403c887198932b308e8d 2024-11-07T15:25:13,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/4e9acc68bd9d45ed95a9e8faf3cbca78 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/4e9acc68bd9d45ed95a9e8faf3cbca78 2024-11-07T15:25:13,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/4e9acc68bd9d45ed95a9e8faf3cbca78, entries=400, sequenceid=182, filesize=72.6 K 2024-11-07T15:25:13,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/874f9af43b394fdea8dac63ac82077d8 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/874f9af43b394fdea8dac63ac82077d8 2024-11-07T15:25:13,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/874f9af43b394fdea8dac63ac82077d8, entries=100, sequenceid=182, filesize=9.5 K 2024-11-07T15:25:13,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/c9b3c2fd7a45403c887198932b308e8d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/c9b3c2fd7a45403c887198932b308e8d 2024-11-07T15:25:13,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/c9b3c2fd7a45403c887198932b308e8d, entries=100, sequenceid=182, filesize=9.5 K 2024-11-07T15:25:13,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 5489735189eabe07c0ddbb2d15b5812f in 1300ms, sequenceid=182, compaction requested=true 2024-11-07T15:25:13,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:13,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:25:13,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:13,519 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:13,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:25:13,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:13,519 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:13,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:25:13,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:13,522 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:13,522 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32007 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:13,522 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/A is initiating minor compaction (all files) 2024-11-07T15:25:13,522 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/B is initiating minor compaction (all files) 2024-11-07T15:25:13,522 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/B in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:13,522 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/A in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:13,522 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/fcd5a51d02ce47ab92016cff28499c2b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/3f4f26ca83794876a74f36884ca93468, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/4e9acc68bd9d45ed95a9e8faf3cbca78] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=125.3 K 2024-11-07T15:25:13,522 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/9b3b99e44def4779a77e45255851f661, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/bce0a27df8a34f2f9340aa963bc21a38, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/874f9af43b394fdea8dac63ac82077d8] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=31.3 K 2024-11-07T15:25:13,522 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:13,522 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/fcd5a51d02ce47ab92016cff28499c2b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/3f4f26ca83794876a74f36884ca93468, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/4e9acc68bd9d45ed95a9e8faf3cbca78] 2024-11-07T15:25:13,522 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcd5a51d02ce47ab92016cff28499c2b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730993109906 2024-11-07T15:25:13,522 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b3b99e44def4779a77e45255851f661, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730993109906 2024-11-07T15:25:13,523 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting bce0a27df8a34f2f9340aa963bc21a38, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730993110028 2024-11-07T15:25:13,523 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f4f26ca83794876a74f36884ca93468, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730993110028 2024-11-07T15:25:13,523 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 874f9af43b394fdea8dac63ac82077d8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1730993112214 2024-11-07T15:25:13,523 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e9acc68bd9d45ed95a9e8faf3cbca78, keycount=400, bloomtype=ROW, size=72.6 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1730993112200 2024-11-07T15:25:13,528 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:13,529 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#B#compaction#475 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:13,529 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/778f374e50ae4a538b2df1fcd09a981e is 50, key is test_row_0/B:col10/1730993112218/Put/seqid=0 2024-11-07T15:25:13,531 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107c89063ea758346f5bad02ea04d202661_5489735189eabe07c0ddbb2d15b5812f store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:13,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742392_1568 (size=12595) 2024-11-07T15:25:13,537 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/778f374e50ae4a538b2df1fcd09a981e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/778f374e50ae4a538b2df1fcd09a981e 2024-11-07T15:25:13,538 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107c89063ea758346f5bad02ea04d202661_5489735189eabe07c0ddbb2d15b5812f, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:13,538 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c89063ea758346f5bad02ea04d202661_5489735189eabe07c0ddbb2d15b5812f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:13,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T15:25:13,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:13,540 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T15:25:13,542 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/B of 5489735189eabe07c0ddbb2d15b5812f into 778f374e50ae4a538b2df1fcd09a981e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:13,542 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:13,542 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/B, priority=13, startTime=1730993113519; duration=0sec 2024-11-07T15:25:13,542 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:13,542 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:B 2024-11-07T15:25:13,542 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:13,544 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32007 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:13,544 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/C is initiating minor compaction (all files) 2024-11-07T15:25:13,544 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/C in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:13,544 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/aef9f8bebb704b92959d429bfdd9d498, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/24766d8c28c24fd2913ee5f31752192c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/c9b3c2fd7a45403c887198932b308e8d] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=31.3 K 2024-11-07T15:25:13,545 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting aef9f8bebb704b92959d429bfdd9d498, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1730993109906 2024-11-07T15:25:13,545 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 24766d8c28c24fd2913ee5f31752192c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1730993110028 2024-11-07T15:25:13,545 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting c9b3c2fd7a45403c887198932b308e8d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1730993112214 2024-11-07T15:25:13,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742393_1569 (size=4469) 2024-11-07T15:25:13,562 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#A#compaction#476 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:13,562 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/6c1a7f27e6a047b3965f199bea48bc65 is 175, key is test_row_0/A:col10/1730993112218/Put/seqid=0 2024-11-07T15:25:13,567 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#C#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:13,567 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/75880788678b4d538174702f437d265e is 50, key is test_row_0/C:col10/1730993112218/Put/seqid=0 2024-11-07T15:25:13,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bca590a482c1477fa28a10b81a18159f_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993112260/Put/seqid=0 2024-11-07T15:25:13,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742395_1571 (size=12595) 2024-11-07T15:25:13,591 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/75880788678b4d538174702f437d265e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/75880788678b4d538174702f437d265e 2024-11-07T15:25:13,595 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/C of 5489735189eabe07c0ddbb2d15b5812f into 75880788678b4d538174702f437d265e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:13,595 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:13,595 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/C, priority=13, startTime=1730993113519; duration=0sec 2024-11-07T15:25:13,595 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:13,595 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:C 2024-11-07T15:25:13,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742394_1570 (size=31656) 2024-11-07T15:25:13,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742396_1572 (size=12304) 2024-11-07T15:25:13,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:13,601 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bca590a482c1477fa28a10b81a18159f_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bca590a482c1477fa28a10b81a18159f_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:13,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/97f833ff0ee842f8b5d4726fb10e2daa, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:13,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/97f833ff0ee842f8b5d4726fb10e2daa is 175, key is test_row_0/A:col10/1730993112260/Put/seqid=0 2024-11-07T15:25:13,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742397_1573 (size=31105) 2024-11-07T15:25:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T15:25:14,001 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/6c1a7f27e6a047b3965f199bea48bc65 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/6c1a7f27e6a047b3965f199bea48bc65 2024-11-07T15:25:14,004 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/A of 5489735189eabe07c0ddbb2d15b5812f into 6c1a7f27e6a047b3965f199bea48bc65(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:14,004 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:14,004 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/A, priority=13, startTime=1730993113519; duration=0sec 2024-11-07T15:25:14,004 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:14,004 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:A 2024-11-07T15:25:14,006 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/97f833ff0ee842f8b5d4726fb10e2daa 2024-11-07T15:25:14,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/224478a57d8043cfa33fc9d79321eca2 is 50, key is test_row_0/B:col10/1730993112260/Put/seqid=0 2024-11-07T15:25:14,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742398_1574 (size=12151) 2024-11-07T15:25:14,414 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/224478a57d8043cfa33fc9d79321eca2 2024-11-07T15:25:14,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/bc7e8a0468a4422eb2e4075f2b09d772 is 50, key is test_row_0/C:col10/1730993112260/Put/seqid=0 2024-11-07T15:25:14,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:14,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993174437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:14,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993174438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:14,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993174439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:14,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993174440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:14,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:14,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742399_1575 (size=12151) 2024-11-07T15:25:14,450 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/bc7e8a0468a4422eb2e4075f2b09d772 2024-11-07T15:25:14,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/97f833ff0ee842f8b5d4726fb10e2daa as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/97f833ff0ee842f8b5d4726fb10e2daa 2024-11-07T15:25:14,456 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/97f833ff0ee842f8b5d4726fb10e2daa, entries=150, sequenceid=208, filesize=30.4 K 2024-11-07T15:25:14,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/224478a57d8043cfa33fc9d79321eca2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/224478a57d8043cfa33fc9d79321eca2 2024-11-07T15:25:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,460 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/224478a57d8043cfa33fc9d79321eca2, entries=150, sequenceid=208, filesize=11.9 K 2024-11-07T15:25:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/bc7e8a0468a4422eb2e4075f2b09d772 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/bc7e8a0468a4422eb2e4075f2b09d772 2024-11-07T15:25:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,464 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/bc7e8a0468a4422eb2e4075f2b09d772, entries=150, sequenceid=208, filesize=11.9 K 2024-11-07T15:25:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,464 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 5489735189eabe07c0ddbb2d15b5812f in 924ms, sequenceid=208, compaction requested=false 2024-11-07T15:25:14,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:14,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:14,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-07T15:25:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-07T15:25:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-07T15:25:14,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9920 sec 2024-11-07T15:25:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.9970 sec 2024-11-07T15:25:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:14,557 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:25:14,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T15:25:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107616da3650db6495fb15c63c946a9b1b5_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_1/A:col10/1730993114438/Put/seqid=0 2024-11-07T15:25:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742400_1576 (size=14794) 2024-11-07T15:25:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993174585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:14,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993174590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:14,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993174590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:14,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993174590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T15:25:14,621 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:25:14,622 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:25:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-07T15:25:14,623 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:25:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T15:25:14,624 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:25:14,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:25:14,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:14,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993174701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993174701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:14,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993174701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993174701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:14,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T15:25:14,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T15:25:14,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:14,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:14,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:14,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:14,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:14,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993174911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:14,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993174911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:14,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:14,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:14,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993174911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:14,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993174911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:14,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:14,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:14,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:14,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T15:25:14,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:14,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:14,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:14,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:14,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T15:25:14,967 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:14,969 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107616da3650db6495fb15c63c946a9b1b5_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107616da3650db6495fb15c63c946a9b1b5_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:14,970 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/b65ccbbccd3e465586a930332442c251, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:14,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/b65ccbbccd3e465586a930332442c251 is 175, key is test_row_1/A:col10/1730993114438/Put/seqid=0 2024-11-07T15:25:14,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742401_1577 (size=39745) 2024-11-07T15:25:15,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T15:25:15,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:15,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:15,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:15,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:15,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:15,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993175220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:15,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993175220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:15,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993175221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:15,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993175221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:15,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T15:25:15,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:15,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:15,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:15,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:15,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T15:25:15,374 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/b65ccbbccd3e465586a930332442c251 2024-11-07T15:25:15,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/28a63483c4b64120b33c376001156a5f is 50, key is test_row_1/B:col10/1730993114438/Put/seqid=0 2024-11-07T15:25:15,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742402_1578 (size=9757) 2024-11-07T15:25:15,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/28a63483c4b64120b33c376001156a5f 2024-11-07T15:25:15,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T15:25:15,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:15,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:15,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:15,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:15,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:15,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/7893eb5c0d0f4ebf8c41a5969c0c4e32 is 50, key is test_row_1/C:col10/1730993114438/Put/seqid=0 2024-11-07T15:25:15,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742403_1579 (size=9757) 2024-11-07T15:25:15,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/7893eb5c0d0f4ebf8c41a5969c0c4e32 2024-11-07T15:25:15,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/b65ccbbccd3e465586a930332442c251 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/b65ccbbccd3e465586a930332442c251 2024-11-07T15:25:15,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/b65ccbbccd3e465586a930332442c251, entries=200, sequenceid=224, filesize=38.8 K 2024-11-07T15:25:15,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/28a63483c4b64120b33c376001156a5f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/28a63483c4b64120b33c376001156a5f 2024-11-07T15:25:15,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/28a63483c4b64120b33c376001156a5f, entries=100, sequenceid=224, filesize=9.5 K 2024-11-07T15:25:15,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/7893eb5c0d0f4ebf8c41a5969c0c4e32 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/7893eb5c0d0f4ebf8c41a5969c0c4e32 2024-11-07T15:25:15,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/7893eb5c0d0f4ebf8c41a5969c0c4e32, entries=100, sequenceid=224, filesize=9.5 K 2024-11-07T15:25:15,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5489735189eabe07c0ddbb2d15b5812f in 856ms, sequenceid=224, compaction requested=true 2024-11-07T15:25:15,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:15,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:25:15,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:15,414 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:15,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:25:15,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:15,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:25:15,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T15:25:15,414 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:15,415 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:15,415 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102506 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:15,415 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/A is initiating minor compaction (all files) 2024-11-07T15:25:15,415 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/B is initiating minor compaction (all files) 2024-11-07T15:25:15,415 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/A in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:15,415 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/B in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:15,415 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/778f374e50ae4a538b2df1fcd09a981e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/224478a57d8043cfa33fc9d79321eca2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/28a63483c4b64120b33c376001156a5f] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=33.7 K 2024-11-07T15:25:15,415 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/6c1a7f27e6a047b3965f199bea48bc65, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/97f833ff0ee842f8b5d4726fb10e2daa, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/b65ccbbccd3e465586a930332442c251] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=100.1 K 2024-11-07T15:25:15,415 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:15,415 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/6c1a7f27e6a047b3965f199bea48bc65, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/97f833ff0ee842f8b5d4726fb10e2daa, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/b65ccbbccd3e465586a930332442c251] 2024-11-07T15:25:15,415 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c1a7f27e6a047b3965f199bea48bc65, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1730993110028 2024-11-07T15:25:15,415 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 778f374e50ae4a538b2df1fcd09a981e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1730993110028 2024-11-07T15:25:15,416 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 224478a57d8043cfa33fc9d79321eca2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730993112260 2024-11-07T15:25:15,416 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97f833ff0ee842f8b5d4726fb10e2daa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730993112260 2024-11-07T15:25:15,416 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 28a63483c4b64120b33c376001156a5f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1730993114438 2024-11-07T15:25:15,416 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting b65ccbbccd3e465586a930332442c251, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1730993114438 2024-11-07T15:25:15,421 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#B#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:15,422 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/5447daa11855461087ddfde917bec5e5 is 50, key is test_row_0/B:col10/1730993112260/Put/seqid=0 2024-11-07T15:25:15,426 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:15,429 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107becd3b9f4cc948f8b1ee831fe1b6c887_5489735189eabe07c0ddbb2d15b5812f store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:15,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742404_1580 (size=12697) 2024-11-07T15:25:15,431 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107becd3b9f4cc948f8b1ee831fe1b6c887_5489735189eabe07c0ddbb2d15b5812f, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:15,431 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107becd3b9f4cc948f8b1ee831fe1b6c887_5489735189eabe07c0ddbb2d15b5812f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:15,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742405_1581 (size=4469) 2024-11-07T15:25:15,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T15:25:15,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:15,539 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T15:25:15,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ed49d9e836c446d596ac0e6eac3b9106_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993114589/Put/seqid=0 2024-11-07T15:25:15,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742406_1582 (size=12304) 2024-11-07T15:25:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:15,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:15,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:15,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993175742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:15,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:15,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993175747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993175748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993175748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T15:25:15,833 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/5447daa11855461087ddfde917bec5e5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/5447daa11855461087ddfde917bec5e5 2024-11-07T15:25:15,836 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#A#compaction#485 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:15,837 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/d12cf0cf642548ea980629c598cda8e5 is 175, key is test_row_0/A:col10/1730993112260/Put/seqid=0 2024-11-07T15:25:15,837 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/B of 5489735189eabe07c0ddbb2d15b5812f into 5447daa11855461087ddfde917bec5e5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:15,837 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:15,838 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/B, priority=13, startTime=1730993115414; duration=0sec 2024-11-07T15:25:15,838 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:15,838 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:B 2024-11-07T15:25:15,838 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:15,838 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:15,839 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/C is initiating minor compaction (all files) 2024-11-07T15:25:15,839 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/C in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:15,839 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/75880788678b4d538174702f437d265e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/bc7e8a0468a4422eb2e4075f2b09d772, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/7893eb5c0d0f4ebf8c41a5969c0c4e32] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=33.7 K 2024-11-07T15:25:15,839 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 75880788678b4d538174702f437d265e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1730993110028 2024-11-07T15:25:15,840 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting bc7e8a0468a4422eb2e4075f2b09d772, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730993112260 2024-11-07T15:25:15,840 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 7893eb5c0d0f4ebf8c41a5969c0c4e32, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1730993114438 2024-11-07T15:25:15,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742407_1583 (size=31758) 2024-11-07T15:25:15,845 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#C#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:15,846 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/773d9324d5d94fcc9ad637851cfb7687 is 50, key is test_row_0/C:col10/1730993112260/Put/seqid=0 2024-11-07T15:25:15,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993175850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:15,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:15,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742408_1584 (size=12697) 2024-11-07T15:25:15,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993175861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:15,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:15,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993175861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:15,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:15,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993175861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:15,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:15,953 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ed49d9e836c446d596ac0e6eac3b9106_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ed49d9e836c446d596ac0e6eac3b9106_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:15,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/c6a54e2301114e428e9a50851eb04908, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:15,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/c6a54e2301114e428e9a50851eb04908 is 175, key is test_row_0/A:col10/1730993114589/Put/seqid=0 2024-11-07T15:25:15,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742409_1585 (size=31105) 2024-11-07T15:25:15,957 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/c6a54e2301114e428e9a50851eb04908 2024-11-07T15:25:15,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/844c1ec7e2ae47cbaa3be1e8d2045ae1 is 50, key is test_row_0/B:col10/1730993114589/Put/seqid=0 2024-11-07T15:25:15,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742410_1586 (size=12151) 2024-11-07T15:25:16,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:16,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993176060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:16,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993176071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:16,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993176071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:16,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993176071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:16,246 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/d12cf0cf642548ea980629c598cda8e5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/d12cf0cf642548ea980629c598cda8e5 2024-11-07T15:25:16,250 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/A of 5489735189eabe07c0ddbb2d15b5812f into d12cf0cf642548ea980629c598cda8e5(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:16,250 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:16,250 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/A, priority=13, startTime=1730993115413; duration=0sec 2024-11-07T15:25:16,250 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:16,250 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:A 2024-11-07T15:25:16,258 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/773d9324d5d94fcc9ad637851cfb7687 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/773d9324d5d94fcc9ad637851cfb7687 2024-11-07T15:25:16,261 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/C of 5489735189eabe07c0ddbb2d15b5812f into 773d9324d5d94fcc9ad637851cfb7687(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:16,261 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:16,261 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/C, priority=13, startTime=1730993115414; duration=0sec 2024-11-07T15:25:16,262 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:16,262 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:C 2024-11-07T15:25:16,367 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/844c1ec7e2ae47cbaa3be1e8d2045ae1 2024-11-07T15:25:16,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/071cba4525cc4945b6f6118709fd414e is 50, key is test_row_0/C:col10/1730993114589/Put/seqid=0 2024-11-07T15:25:16,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993176371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:16,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:16,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742411_1587 (size=12151) 2024-11-07T15:25:16,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993176380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:16,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:16,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993176380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:16,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993176380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:16,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:16,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:16,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58552 deadline: 1730993176401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:16,402 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7b3e4e8e12ef,33849,1730992951906 for 'test_row_2' in 5489735189eabe07c0ddbb2d15b5812f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-07T15:25:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:16,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T15:25:16,776 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/071cba4525cc4945b6f6118709fd414e 2024-11-07T15:25:16,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/c6a54e2301114e428e9a50851eb04908 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/c6a54e2301114e428e9a50851eb04908 2024-11-07T15:25:16,782 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/c6a54e2301114e428e9a50851eb04908, entries=150, sequenceid=247, filesize=30.4 K 2024-11-07T15:25:16,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/844c1ec7e2ae47cbaa3be1e8d2045ae1 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/844c1ec7e2ae47cbaa3be1e8d2045ae1 2024-11-07T15:25:16,785 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/844c1ec7e2ae47cbaa3be1e8d2045ae1, entries=150, sequenceid=247, filesize=11.9 K 2024-11-07T15:25:16,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/071cba4525cc4945b6f6118709fd414e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/071cba4525cc4945b6f6118709fd414e 2024-11-07T15:25:16,788 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/071cba4525cc4945b6f6118709fd414e, entries=150, sequenceid=247, filesize=11.9 K 2024-11-07T15:25:16,789 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 5489735189eabe07c0ddbb2d15b5812f in 1250ms, sequenceid=247, compaction requested=false 2024-11-07T15:25:16,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:16,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:16,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-07T15:25:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-07T15:25:16,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-07T15:25:16,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1660 sec 2024-11-07T15:25:16,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.1700 sec 2024-11-07T15:25:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:16,882 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:25:16,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T15:25:16,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110765766928d41f4892a79f11bd7b9856b5_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993115747/Put/seqid=0 2024-11-07T15:25:16,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742412_1588 (size=12454) 2024-11-07T15:25:16,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:16,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993176907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:16,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:16,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993176907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:16,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:16,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:16,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993176911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:16,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993176911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:16,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:16,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:16,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:17,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:17,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993177021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993177021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993177021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:17,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993177021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:17,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993177231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:17,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993177231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993177231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993177231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:17,294 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:17,297 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110765766928d41f4892a79f11bd7b9856b5_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110765766928d41f4892a79f11bd7b9856b5_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:17,297 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/e71e7a43178f4e0183bdca42da532963, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:17,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/e71e7a43178f4e0183bdca42da532963 is 175, key is test_row_0/A:col10/1730993115747/Put/seqid=0 2024-11-07T15:25:17,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742413_1589 (size=31255) 2024-11-07T15:25:17,301 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/e71e7a43178f4e0183bdca42da532963 2024-11-07T15:25:17,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/f9e120d0e5ff4380afe70dbf4925cf05 is 50, key is test_row_0/B:col10/1730993115747/Put/seqid=0 2024-11-07T15:25:17,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742414_1590 (size=12301) 2024-11-07T15:25:17,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:17,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993177540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:17,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:17,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993177541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:17,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:17,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:17,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:17,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993177541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:17,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:17,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993177541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:17,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:17,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:17,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:17,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/f9e120d0e5ff4380afe70dbf4925cf05 2024-11-07T15:25:17,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/d8bf8b10419b4c85b4a2fc8c0edc9056 is 50, key is test_row_0/C:col10/1730993115747/Put/seqid=0 2024-11-07T15:25:17,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742415_1591 (size=12301) 2024-11-07T15:25:17,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/d8bf8b10419b4c85b4a2fc8c0edc9056 2024-11-07T15:25:17,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/e71e7a43178f4e0183bdca42da532963 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/e71e7a43178f4e0183bdca42da532963 2024-11-07T15:25:17,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/e71e7a43178f4e0183bdca42da532963, entries=150, sequenceid=265, filesize=30.5 K 2024-11-07T15:25:17,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/f9e120d0e5ff4380afe70dbf4925cf05 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/f9e120d0e5ff4380afe70dbf4925cf05 2024-11-07T15:25:17,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/f9e120d0e5ff4380afe70dbf4925cf05, entries=150, sequenceid=265, filesize=12.0 K 2024-11-07T15:25:17,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/d8bf8b10419b4c85b4a2fc8c0edc9056 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/d8bf8b10419b4c85b4a2fc8c0edc9056 2024-11-07T15:25:17,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/d8bf8b10419b4c85b4a2fc8c0edc9056, entries=150, sequenceid=265, filesize=12.0 K 2024-11-07T15:25:17,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5489735189eabe07c0ddbb2d15b5812f in 874ms, sequenceid=265, compaction requested=true 2024-11-07T15:25:17,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:17,756 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:17,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:25:17,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:17,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:25:17,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:17,756 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:17,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:25:17,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:17,757 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94118 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:17,757 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/A is initiating minor compaction (all files) 2024-11-07T15:25:17,757 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/A in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:17,757 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/d12cf0cf642548ea980629c598cda8e5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/c6a54e2301114e428e9a50851eb04908, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/e71e7a43178f4e0183bdca42da532963] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=91.9 K 2024-11-07T15:25:17,757 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:17,757 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/d12cf0cf642548ea980629c598cda8e5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/c6a54e2301114e428e9a50851eb04908, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/e71e7a43178f4e0183bdca42da532963] 2024-11-07T15:25:17,757 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:17,757 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/B is initiating minor compaction (all files) 2024-11-07T15:25:17,757 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting d12cf0cf642548ea980629c598cda8e5, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1730993112260 2024-11-07T15:25:17,757 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/B in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:17,758 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/5447daa11855461087ddfde917bec5e5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/844c1ec7e2ae47cbaa3be1e8d2045ae1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/f9e120d0e5ff4380afe70dbf4925cf05] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=36.3 K 2024-11-07T15:25:17,758 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6a54e2301114e428e9a50851eb04908, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1730993114584 2024-11-07T15:25:17,758 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 5447daa11855461087ddfde917bec5e5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1730993112260 2024-11-07T15:25:17,758 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting e71e7a43178f4e0183bdca42da532963, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730993115746 2024-11-07T15:25:17,758 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 844c1ec7e2ae47cbaa3be1e8d2045ae1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1730993114584 2024-11-07T15:25:17,759 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting f9e120d0e5ff4380afe70dbf4925cf05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730993115746 2024-11-07T15:25:17,777 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:17,789 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#B#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:17,789 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/0473aa3a6132493e9e0897e1e9aad1c2 is 50, key is test_row_0/B:col10/1730993115747/Put/seqid=0 2024-11-07T15:25:17,795 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107fb18aa355c424fd4841b06cefe8c4b35_5489735189eabe07c0ddbb2d15b5812f store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:17,798 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107fb18aa355c424fd4841b06cefe8c4b35_5489735189eabe07c0ddbb2d15b5812f, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:17,798 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107fb18aa355c424fd4841b06cefe8c4b35_5489735189eabe07c0ddbb2d15b5812f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:17,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742416_1592 (size=12949) 2024-11-07T15:25:17,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742417_1593 (size=4469) 2024-11-07T15:25:17,851 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#A#compaction#493 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:17,851 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/2a5f9601bcac48a484e2a39cc5bd9625 is 175, key is test_row_0/A:col10/1730993115747/Put/seqid=0 2024-11-07T15:25:17,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742418_1594 (size=31903) 2024-11-07T15:25:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:18,054 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:25:18,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T15:25:18,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110796dc29f457f44c2fa2f9bb2922facfc3_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993118053/Put/seqid=0 2024-11-07T15:25:18,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742419_1595 (size=12454) 2024-11-07T15:25:18,065 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:18,069 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110796dc29f457f44c2fa2f9bb2922facfc3_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110796dc29f457f44c2fa2f9bb2922facfc3_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:18,071 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/e67fc72b74184bf9bfeb0ac0652bedb2, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:18,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/e67fc72b74184bf9bfeb0ac0652bedb2 is 175, key is test_row_0/A:col10/1730993118053/Put/seqid=0 2024-11-07T15:25:18,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993178068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993178067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993178069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993178069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742420_1596 (size=31255) 2024-11-07T15:25:18,078 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/e67fc72b74184bf9bfeb0ac0652bedb2 2024-11-07T15:25:18,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/38f4ffc5cf3b4bbab399c802ce5f832d is 50, key is test_row_0/B:col10/1730993118053/Put/seqid=0 2024-11-07T15:25:18,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742421_1597 (size=12301) 2024-11-07T15:25:18,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/38f4ffc5cf3b4bbab399c802ce5f832d 2024-11-07T15:25:18,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/f2d8e811235b4f179df896e61dbf7bd2 is 50, key is test_row_0/C:col10/1730993118053/Put/seqid=0 2024-11-07T15:25:18,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742422_1598 (size=12301) 2024-11-07T15:25:18,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993178181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993178181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993178181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993178181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,251 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/0473aa3a6132493e9e0897e1e9aad1c2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/0473aa3a6132493e9e0897e1e9aad1c2 2024-11-07T15:25:18,256 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/B of 5489735189eabe07c0ddbb2d15b5812f into 0473aa3a6132493e9e0897e1e9aad1c2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:18,256 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:18,256 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/B, priority=13, startTime=1730993117756; duration=0sec 2024-11-07T15:25:18,256 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:18,256 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:B 2024-11-07T15:25:18,256 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:18,257 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:18,257 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/C is initiating minor compaction (all files) 2024-11-07T15:25:18,257 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/C in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:18,257 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/773d9324d5d94fcc9ad637851cfb7687, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/071cba4525cc4945b6f6118709fd414e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/d8bf8b10419b4c85b4a2fc8c0edc9056] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=36.3 K 2024-11-07T15:25:18,258 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 773d9324d5d94fcc9ad637851cfb7687, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1730993112260 2024-11-07T15:25:18,258 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 071cba4525cc4945b6f6118709fd414e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1730993114584 2024-11-07T15:25:18,258 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting d8bf8b10419b4c85b4a2fc8c0edc9056, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730993115746 2024-11-07T15:25:18,265 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#C#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:18,266 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/9610029e87e644519583b03092327bc5 is 50, key is test_row_0/C:col10/1730993115747/Put/seqid=0 2024-11-07T15:25:18,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742423_1599 (size=12949) 2024-11-07T15:25:18,284 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/9610029e87e644519583b03092327bc5 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/9610029e87e644519583b03092327bc5 2024-11-07T15:25:18,288 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/C of 5489735189eabe07c0ddbb2d15b5812f into 9610029e87e644519583b03092327bc5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:18,288 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:18,288 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/C, priority=13, startTime=1730993117756; duration=0sec 2024-11-07T15:25:18,288 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:18,288 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:C 2024-11-07T15:25:18,320 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/2a5f9601bcac48a484e2a39cc5bd9625 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/2a5f9601bcac48a484e2a39cc5bd9625 2024-11-07T15:25:18,326 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/A of 5489735189eabe07c0ddbb2d15b5812f into 2a5f9601bcac48a484e2a39cc5bd9625(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:18,326 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:18,326 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/A, priority=13, startTime=1730993117756; duration=0sec 2024-11-07T15:25:18,326 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:18,326 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:A 2024-11-07T15:25:18,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993178391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993178392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993178392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993178392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/f2d8e811235b4f179df896e61dbf7bd2 2024-11-07T15:25:18,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/e67fc72b74184bf9bfeb0ac0652bedb2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/e67fc72b74184bf9bfeb0ac0652bedb2 2024-11-07T15:25:18,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/e67fc72b74184bf9bfeb0ac0652bedb2, entries=150, sequenceid=288, filesize=30.5 K 2024-11-07T15:25:18,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/38f4ffc5cf3b4bbab399c802ce5f832d as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/38f4ffc5cf3b4bbab399c802ce5f832d 2024-11-07T15:25:18,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/38f4ffc5cf3b4bbab399c802ce5f832d, entries=150, sequenceid=288, filesize=12.0 K 2024-11-07T15:25:18,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/f2d8e811235b4f179df896e61dbf7bd2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/f2d8e811235b4f179df896e61dbf7bd2 2024-11-07T15:25:18,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/f2d8e811235b4f179df896e61dbf7bd2, entries=150, sequenceid=288, filesize=12.0 K 2024-11-07T15:25:18,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 5489735189eabe07c0ddbb2d15b5812f in 466ms, sequenceid=288, compaction requested=false 2024-11-07T15:25:18,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:18,703 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:25:18,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T15:25:18,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110794308e8e53d14d16af0d0d40ac279ebc_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993118702/Put/seqid=0 2024-11-07T15:25:18,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742424_1600 (size=12454) 2024-11-07T15:25:18,731 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:18,735 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110794308e8e53d14d16af0d0d40ac279ebc_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110794308e8e53d14d16af0d0d40ac279ebc_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:18,736 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/3361cd8c29404a69b0431f246e3867a8, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:18,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/3361cd8c29404a69b0431f246e3867a8 is 175, key is test_row_0/A:col10/1730993118702/Put/seqid=0 2024-11-07T15:25:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993178730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993178730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993178736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993178737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742425_1601 (size=31255) 2024-11-07T15:25:18,768 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/3361cd8c29404a69b0431f246e3867a8 2024-11-07T15:25:18,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/a45a5915171b48e3aae541af71e590c2 is 50, key is test_row_0/B:col10/1730993118702/Put/seqid=0 2024-11-07T15:25:18,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742426_1602 (size=12301) 2024-11-07T15:25:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T15:25:18,791 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:25:18,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/a45a5915171b48e3aae541af71e590c2 2024-11-07T15:25:18,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:25:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-07T15:25:18,794 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:25:18,795 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:25:18,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:25:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T15:25:18,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/1d6331bdca6e41d2bed25424c085932a is 50, key is test_row_0/C:col10/1730993118702/Put/seqid=0 2024-11-07T15:25:18,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742427_1603 (size=12301) 2024-11-07T15:25:18,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993178841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993178841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993178851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993178851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T15:25:18,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-07T15:25:18,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:18,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:18,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:18,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:18,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:19,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993179050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:19,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993179050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:19,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993179061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993179061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:19,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-07T15:25:19,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:19,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:19,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:19,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:19,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T15:25:19,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/1d6331bdca6e41d2bed25424c085932a 2024-11-07T15:25:19,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/3361cd8c29404a69b0431f246e3867a8 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/3361cd8c29404a69b0431f246e3867a8 2024-11-07T15:25:19,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/3361cd8c29404a69b0431f246e3867a8, entries=150, sequenceid=306, filesize=30.5 K 2024-11-07T15:25:19,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/a45a5915171b48e3aae541af71e590c2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/a45a5915171b48e3aae541af71e590c2 2024-11-07T15:25:19,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/a45a5915171b48e3aae541af71e590c2, entries=150, sequenceid=306, filesize=12.0 K 2024-11-07T15:25:19,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/1d6331bdca6e41d2bed25424c085932a as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1d6331bdca6e41d2bed25424c085932a 2024-11-07T15:25:19,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1d6331bdca6e41d2bed25424c085932a, entries=150, sequenceid=306, filesize=12.0 K 2024-11-07T15:25:19,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 5489735189eabe07c0ddbb2d15b5812f in 521ms, sequenceid=306, compaction requested=true 2024-11-07T15:25:19,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:19,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:25:19,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:19,224 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:19,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:25:19,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:19,224 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:19,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:25:19,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:19,225 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:19,225 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:19,225 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/B is initiating minor compaction (all files) 2024-11-07T15:25:19,225 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/A is initiating minor compaction (all files) 2024-11-07T15:25:19,225 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/B in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:19,225 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/A in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:19,225 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/0473aa3a6132493e9e0897e1e9aad1c2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/38f4ffc5cf3b4bbab399c802ce5f832d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/a45a5915171b48e3aae541af71e590c2] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=36.7 K 2024-11-07T15:25:19,225 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/2a5f9601bcac48a484e2a39cc5bd9625, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/e67fc72b74184bf9bfeb0ac0652bedb2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/3361cd8c29404a69b0431f246e3867a8] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=92.2 K 2024-11-07T15:25:19,225 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:19,225 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/2a5f9601bcac48a484e2a39cc5bd9625, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/e67fc72b74184bf9bfeb0ac0652bedb2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/3361cd8c29404a69b0431f246e3867a8] 2024-11-07T15:25:19,226 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a5f9601bcac48a484e2a39cc5bd9625, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730993115746 2024-11-07T15:25:19,226 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 0473aa3a6132493e9e0897e1e9aad1c2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730993115746 2024-11-07T15:25:19,226 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 38f4ffc5cf3b4bbab399c802ce5f832d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730993116906 2024-11-07T15:25:19,226 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting e67fc72b74184bf9bfeb0ac0652bedb2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730993116906 2024-11-07T15:25:19,227 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting a45a5915171b48e3aae541af71e590c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1730993118054 2024-11-07T15:25:19,227 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3361cd8c29404a69b0431f246e3867a8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1730993118054 2024-11-07T15:25:19,232 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:19,233 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#B#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:19,234 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/0cb6390362ea4017be3530dd547d3b76 is 50, key is test_row_0/B:col10/1730993118702/Put/seqid=0 2024-11-07T15:25:19,235 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110785b7f463b83641d5a4a306472df0782d_5489735189eabe07c0ddbb2d15b5812f store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:19,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742428_1604 (size=13051) 2024-11-07T15:25:19,238 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110785b7f463b83641d5a4a306472df0782d_5489735189eabe07c0ddbb2d15b5812f, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:19,238 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110785b7f463b83641d5a4a306472df0782d_5489735189eabe07c0ddbb2d15b5812f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:19,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742429_1605 (size=4469) 2024-11-07T15:25:19,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-07T15:25:19,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:19,253 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T15:25:19,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110721b3b4d09cd64073a91412f65c6ee1f8_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993118734/Put/seqid=0 2024-11-07T15:25:19,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742430_1606 (size=12454) 2024-11-07T15:25:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:19,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:19,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993179381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:19,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993179381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:19,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993179382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993179382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T15:25:19,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993179490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:19,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993179490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:19,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993179491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993179491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:19,642 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/0cb6390362ea4017be3530dd547d3b76 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/0cb6390362ea4017be3530dd547d3b76 2024-11-07T15:25:19,642 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#A#compaction#502 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:19,642 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/7d1dd292dffc491f846a1ccdc95ca3d7 is 175, key is test_row_0/A:col10/1730993118702/Put/seqid=0 2024-11-07T15:25:19,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742431_1607 (size=32005) 2024-11-07T15:25:19,649 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/B of 5489735189eabe07c0ddbb2d15b5812f into 0cb6390362ea4017be3530dd547d3b76(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:19,649 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:19,649 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/B, priority=13, startTime=1730993119224; duration=0sec 2024-11-07T15:25:19,649 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:19,649 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:B 2024-11-07T15:25:19,649 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:19,653 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:19,654 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/C is initiating minor compaction (all files) 2024-11-07T15:25:19,654 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/C in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:19,654 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/9610029e87e644519583b03092327bc5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/f2d8e811235b4f179df896e61dbf7bd2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1d6331bdca6e41d2bed25424c085932a] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=36.7 K 2024-11-07T15:25:19,654 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/7d1dd292dffc491f846a1ccdc95ca3d7 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/7d1dd292dffc491f846a1ccdc95ca3d7 2024-11-07T15:25:19,654 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 9610029e87e644519583b03092327bc5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730993115746 2024-11-07T15:25:19,654 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting f2d8e811235b4f179df896e61dbf7bd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1730993116906 2024-11-07T15:25:19,655 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d6331bdca6e41d2bed25424c085932a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1730993118054 2024-11-07T15:25:19,658 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/A of 5489735189eabe07c0ddbb2d15b5812f into 7d1dd292dffc491f846a1ccdc95ca3d7(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:19,658 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:19,658 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/A, priority=13, startTime=1730993119224; duration=0sec 2024-11-07T15:25:19,658 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:19,658 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:A 2024-11-07T15:25:19,661 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#C#compaction#505 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:19,661 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/fb8ddb316d914096bff63cce566438b7 is 50, key is test_row_0/C:col10/1730993118702/Put/seqid=0 2024-11-07T15:25:19,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:19,665 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110721b3b4d09cd64073a91412f65c6ee1f8_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110721b3b4d09cd64073a91412f65c6ee1f8_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:19,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/dbbc6aa31cee4fcd9ae16a2bd3f05106, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:19,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/dbbc6aa31cee4fcd9ae16a2bd3f05106 is 175, key is test_row_0/A:col10/1730993118734/Put/seqid=0 2024-11-07T15:25:19,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742432_1608 (size=13051) 2024-11-07T15:25:19,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742433_1609 (size=31255) 2024-11-07T15:25:19,671 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/dbbc6aa31cee4fcd9ae16a2bd3f05106 2024-11-07T15:25:19,674 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/fb8ddb316d914096bff63cce566438b7 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/fb8ddb316d914096bff63cce566438b7 2024-11-07T15:25:19,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/6da4a90563a04153a09d31377f903588 is 50, key is test_row_0/B:col10/1730993118734/Put/seqid=0 2024-11-07T15:25:19,682 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/C of 5489735189eabe07c0ddbb2d15b5812f into fb8ddb316d914096bff63cce566438b7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:19,682 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:19,682 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/C, priority=13, startTime=1730993119224; duration=0sec 2024-11-07T15:25:19,682 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:19,682 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:C 2024-11-07T15:25:19,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742434_1610 (size=12301) 2024-11-07T15:25:19,686 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/6da4a90563a04153a09d31377f903588 2024-11-07T15:25:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/167e7da5d8604a31b88acc5084ca4a7c is 50, key is test_row_0/C:col10/1730993118734/Put/seqid=0 2024-11-07T15:25:19,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742435_1611 (size=12301) 2024-11-07T15:25:19,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993179701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:19,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993179701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:19,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993179701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:19,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993179701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:19,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T15:25:20,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993180011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:20,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993180011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:20,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993180011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:20,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993180011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:20,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:20,096 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/167e7da5d8604a31b88acc5084ca4a7c 2024-11-07T15:25:20,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/dbbc6aa31cee4fcd9ae16a2bd3f05106 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/dbbc6aa31cee4fcd9ae16a2bd3f05106 2024-11-07T15:25:20,107 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/dbbc6aa31cee4fcd9ae16a2bd3f05106, entries=150, sequenceid=328, filesize=30.5 K 2024-11-07T15:25:20,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/6da4a90563a04153a09d31377f903588 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/6da4a90563a04153a09d31377f903588 2024-11-07T15:25:20,113 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/6da4a90563a04153a09d31377f903588, entries=150, sequenceid=328, filesize=12.0 K 2024-11-07T15:25:20,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/167e7da5d8604a31b88acc5084ca4a7c as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/167e7da5d8604a31b88acc5084ca4a7c 2024-11-07T15:25:20,120 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/167e7da5d8604a31b88acc5084ca4a7c, entries=150, sequenceid=328, filesize=12.0 K 2024-11-07T15:25:20,121 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 5489735189eabe07c0ddbb2d15b5812f in 868ms, sequenceid=328, compaction requested=false 2024-11-07T15:25:20,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:20,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:20,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-07T15:25:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-07T15:25:20,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-07T15:25:20,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3280 sec 2024-11-07T15:25:20,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.3320 sec 2024-11-07T15:25:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:20,523 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:25:20,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-07T15:25:20,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b0da33757a0c413ba1c393fdb99fc642_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993119381/Put/seqid=0 2024-11-07T15:25:20,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742436_1612 (size=12454) 2024-11-07T15:25:20,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993180544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:20,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993180547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:20,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993180548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:20,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993180550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:20,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993180660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993180661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:20,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:20,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993180661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:20,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:20,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993180661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:20,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:20,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993180871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:20,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993180871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:20,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993180871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:20,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993180871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:20,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:20,934 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:20,936 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b0da33757a0c413ba1c393fdb99fc642_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b0da33757a0c413ba1c393fdb99fc642_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:20,937 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/761cff243da1437bb1c75c2528751f9f, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:20,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/761cff243da1437bb1c75c2528751f9f is 175, key is test_row_0/A:col10/1730993119381/Put/seqid=0 2024-11-07T15:25:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T15:25:20,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742437_1613 (size=31255) 2024-11-07T15:25:20,941 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:25:20,941 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/761cff243da1437bb1c75c2528751f9f 2024-11-07T15:25:20,941 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:25:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-07T15:25:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T15:25:20,943 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:25:20,944 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:25:20,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:25:20,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/ec4ad6e9a157404792f477161cf59fef is 50, key is test_row_0/B:col10/1730993119381/Put/seqid=0 2024-11-07T15:25:20,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742438_1614 (size=12301) 2024-11-07T15:25:20,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/ec4ad6e9a157404792f477161cf59fef 2024-11-07T15:25:20,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/259e5d3242bf4706afb8937c7c343efa is 50, key is test_row_0/C:col10/1730993119381/Put/seqid=0 2024-11-07T15:25:20,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742439_1615 (size=12301) 2024-11-07T15:25:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T15:25:21,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-07T15:25:21,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:21,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:21,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:21,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:21,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:21,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:21,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993181180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:21,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993181180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:21,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993181180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:21,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993181181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:21,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-07T15:25:21,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:21,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:21,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:21,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:21,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:25:21,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T15:25:21,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/259e5d3242bf4706afb8937c7c343efa 2024-11-07T15:25:21,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/761cff243da1437bb1c75c2528751f9f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/761cff243da1437bb1c75c2528751f9f 2024-11-07T15:25:21,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/761cff243da1437bb1c75c2528751f9f, entries=150, sequenceid=348, filesize=30.5 K 2024-11-07T15:25:21,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/ec4ad6e9a157404792f477161cf59fef as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/ec4ad6e9a157404792f477161cf59fef 2024-11-07T15:25:21,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/ec4ad6e9a157404792f477161cf59fef, entries=150, sequenceid=348, filesize=12.0 K 2024-11-07T15:25:21,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/259e5d3242bf4706afb8937c7c343efa as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/259e5d3242bf4706afb8937c7c343efa 2024-11-07T15:25:21,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/259e5d3242bf4706afb8937c7c343efa, entries=150, sequenceid=348, filesize=12.0 K 2024-11-07T15:25:21,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5489735189eabe07c0ddbb2d15b5812f in 867ms, sequenceid=348, compaction requested=true 2024-11-07T15:25:21,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:21,391 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:25:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:25:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:21,391 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:25:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:21,392 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:21,392 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:21,392 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/A is initiating minor compaction (all files) 2024-11-07T15:25:21,392 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/B is initiating minor compaction (all files) 2024-11-07T15:25:21,392 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/B in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:21,392 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/A in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:21,392 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/7d1dd292dffc491f846a1ccdc95ca3d7, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/dbbc6aa31cee4fcd9ae16a2bd3f05106, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/761cff243da1437bb1c75c2528751f9f] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=92.3 K 2024-11-07T15:25:21,392 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/0cb6390362ea4017be3530dd547d3b76, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/6da4a90563a04153a09d31377f903588, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/ec4ad6e9a157404792f477161cf59fef] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=36.8 K 2024-11-07T15:25:21,392 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:21,392 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/7d1dd292dffc491f846a1ccdc95ca3d7, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/dbbc6aa31cee4fcd9ae16a2bd3f05106, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/761cff243da1437bb1c75c2528751f9f] 2024-11-07T15:25:21,393 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cb6390362ea4017be3530dd547d3b76, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1730993118054 2024-11-07T15:25:21,393 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d1dd292dffc491f846a1ccdc95ca3d7, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1730993118054 2024-11-07T15:25:21,393 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 6da4a90563a04153a09d31377f903588, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730993118725 2024-11-07T15:25:21,393 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting ec4ad6e9a157404792f477161cf59fef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1730993119381 2024-11-07T15:25:21,393 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbbc6aa31cee4fcd9ae16a2bd3f05106, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730993118725 2024-11-07T15:25:21,394 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 761cff243da1437bb1c75c2528751f9f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1730993119381 2024-11-07T15:25:21,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33849 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-07T15:25:21,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:21,402 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T15:25:21,422 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:21,422 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#B#compaction#511 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:21,423 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/a3926ca7715b4750ac556862ff27c21a is 50, key is test_row_0/B:col10/1730993119381/Put/seqid=0 2024-11-07T15:25:21,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078245940a5a6e45c991139f55b5c4d552_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993120546/Put/seqid=0 2024-11-07T15:25:21,433 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107f49d9f87c77648dfbcfdc786dd3c92ac_5489735189eabe07c0ddbb2d15b5812f store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:21,435 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107f49d9f87c77648dfbcfdc786dd3c92ac_5489735189eabe07c0ddbb2d15b5812f, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:21,435 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f49d9f87c77648dfbcfdc786dd3c92ac_5489735189eabe07c0ddbb2d15b5812f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:21,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742440_1616 (size=12454) 2024-11-07T15:25:21,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742442_1618 (size=4469) 2024-11-07T15:25:21,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742441_1617 (size=13153) 2024-11-07T15:25:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T15:25:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:21,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. as already flushing 2024-11-07T15:25:21,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993181708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:21,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993181709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:21,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993181711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:21,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993181711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:21,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993181821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993181821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:21,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993181821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:21,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993181821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:21,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:21,852 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078245940a5a6e45c991139f55b5c4d552_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078245940a5a6e45c991139f55b5c4d552_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:21,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/9adbff1eab2b4b999028685eb7f30190, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:21,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/9adbff1eab2b4b999028685eb7f30190 is 175, key is test_row_0/A:col10/1730993120546/Put/seqid=0 2024-11-07T15:25:21,857 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#A#compaction#512 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:21,858 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/c5172b10fe994653bdd422bf28d1510f is 175, key is test_row_0/A:col10/1730993119381/Put/seqid=0 2024-11-07T15:25:21,863 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/a3926ca7715b4750ac556862ff27c21a as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/a3926ca7715b4750ac556862ff27c21a 2024-11-07T15:25:21,868 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/B of 5489735189eabe07c0ddbb2d15b5812f into a3926ca7715b4750ac556862ff27c21a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:21,868 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:21,868 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/B, priority=13, startTime=1730993121391; duration=0sec 2024-11-07T15:25:21,869 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:21,869 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:B 2024-11-07T15:25:21,869 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:21,869 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:21,869 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/C is initiating minor compaction (all files) 2024-11-07T15:25:21,869 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/C in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:21,870 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/fb8ddb316d914096bff63cce566438b7, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/167e7da5d8604a31b88acc5084ca4a7c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/259e5d3242bf4706afb8937c7c343efa] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=36.8 K 2024-11-07T15:25:21,870 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting fb8ddb316d914096bff63cce566438b7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1730993118054 2024-11-07T15:25:21,870 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 167e7da5d8604a31b88acc5084ca4a7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1730993118725 2024-11-07T15:25:21,870 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 259e5d3242bf4706afb8937c7c343efa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1730993119381 2024-11-07T15:25:21,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742443_1619 (size=31255) 2024-11-07T15:25:21,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742444_1620 (size=32107) 2024-11-07T15:25:21,875 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#C#compaction#514 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:21,876 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/3b51a1d772cd43269664f6ca6b59b115 is 50, key is test_row_0/C:col10/1730993119381/Put/seqid=0 2024-11-07T15:25:21,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742445_1621 (size=13153) 2024-11-07T15:25:21,882 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/3b51a1d772cd43269664f6ca6b59b115 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/3b51a1d772cd43269664f6ca6b59b115 2024-11-07T15:25:21,885 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/C of 5489735189eabe07c0ddbb2d15b5812f into 3b51a1d772cd43269664f6ca6b59b115(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:21,885 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:21,885 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/C, priority=13, startTime=1730993121391; duration=0sec 2024-11-07T15:25:21,885 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:21,885 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:C 2024-11-07T15:25:22,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993182031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993182031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:22,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993182031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:22,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:22,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993182031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T15:25:22,274 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/9adbff1eab2b4b999028685eb7f30190 2024-11-07T15:25:22,277 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/c5172b10fe994653bdd422bf28d1510f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/c5172b10fe994653bdd422bf28d1510f 2024-11-07T15:25:22,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/c2c84396fe264b3581092dd7af834644 is 50, key is test_row_0/B:col10/1730993120546/Put/seqid=0 2024-11-07T15:25:22,282 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/A of 5489735189eabe07c0ddbb2d15b5812f into c5172b10fe994653bdd422bf28d1510f(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:22,282 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:22,282 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/A, priority=13, startTime=1730993121390; duration=0sec 2024-11-07T15:25:22,282 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:22,282 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:A 2024-11-07T15:25:22,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742446_1622 (size=12301) 2024-11-07T15:25:22,308 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/c2c84396fe264b3581092dd7af834644 2024-11-07T15:25:22,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/3fe38e7b43ef49f7a44f292bd921751f is 50, key is test_row_0/C:col10/1730993120546/Put/seqid=0 2024-11-07T15:25:22,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742447_1623 (size=12301) 2024-11-07T15:25:22,320 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/3fe38e7b43ef49f7a44f292bd921751f 2024-11-07T15:25:22,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/9adbff1eab2b4b999028685eb7f30190 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/9adbff1eab2b4b999028685eb7f30190 2024-11-07T15:25:22,328 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/9adbff1eab2b4b999028685eb7f30190, entries=150, sequenceid=367, filesize=30.5 K 2024-11-07T15:25:22,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/c2c84396fe264b3581092dd7af834644 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/c2c84396fe264b3581092dd7af834644 2024-11-07T15:25:22,332 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/c2c84396fe264b3581092dd7af834644, entries=150, sequenceid=367, filesize=12.0 K 2024-11-07T15:25:22,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/3fe38e7b43ef49f7a44f292bd921751f as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/3fe38e7b43ef49f7a44f292bd921751f 2024-11-07T15:25:22,336 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/3fe38e7b43ef49f7a44f292bd921751f, entries=150, sequenceid=367, filesize=12.0 K 2024-11-07T15:25:22,336 INFO [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5489735189eabe07c0ddbb2d15b5812f in 934ms, sequenceid=367, compaction requested=false 2024-11-07T15:25:22,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:22,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:22,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7b3e4e8e12ef:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-07T15:25:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-07T15:25:22,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-07T15:25:22,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3930 sec 2024-11-07T15:25:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:22,343 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:25:22,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T15:25:22,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.3990 sec 2024-11-07T15:25:22,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c4cbf896f8514f0ba5b9b5a712b41646_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993121710/Put/seqid=0 2024-11-07T15:25:22,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742448_1624 (size=17534) 2024-11-07T15:25:22,354 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:22,356 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c4cbf896f8514f0ba5b9b5a712b41646_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c4cbf896f8514f0ba5b9b5a712b41646_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:22,357 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/378e6602c4514f919d3c0ac993f4fdf2, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:22,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/378e6602c4514f919d3c0ac993f4fdf2 is 175, key is test_row_0/A:col10/1730993121710/Put/seqid=0 2024-11-07T15:25:22,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742449_1625 (size=48639) 2024-11-07T15:25:22,362 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=389, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/378e6602c4514f919d3c0ac993f4fdf2 2024-11-07T15:25:22,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/833ff6abd57e43788ad7a26dbae4d882 is 50, key is test_row_0/B:col10/1730993121710/Put/seqid=0 2024-11-07T15:25:22,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:22,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993182361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993182361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:22,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993182362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:22,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993182369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:22,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:22,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742450_1626 (size=12301) 2024-11-07T15:25:22,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/833ff6abd57e43788ad7a26dbae4d882 2024-11-07T15:25:22,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/e8f4e607f88c472a817f2367880914cb is 50, key is test_row_0/C:col10/1730993121710/Put/seqid=0 2024-11-07T15:25:22,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742451_1627 (size=12301) 2024-11-07T15:25:22,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993182470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:22,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:22,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993182470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:22,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:22,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993182480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:22,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:22,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993182481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:22,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:22,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58542 deadline: 1730993182680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:22,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58480 deadline: 1730993182680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:22,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:22,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:22,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58520 deadline: 1730993182691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:22,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:22,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-07T15:25:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33849 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58532 deadline: 1730993182691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:22,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 , the old value is region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:25:22,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., hostname=7b3e4e8e12ef,33849,1730992951906, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:25:22,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/e8f4e607f88c472a817f2367880914cb 2024-11-07T15:25:22,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/378e6602c4514f919d3c0ac993f4fdf2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/378e6602c4514f919d3c0ac993f4fdf2 2024-11-07T15:25:22,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/378e6602c4514f919d3c0ac993f4fdf2, entries=250, sequenceid=389, filesize=47.5 K 2024-11-07T15:25:22,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/833ff6abd57e43788ad7a26dbae4d882 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/833ff6abd57e43788ad7a26dbae4d882 2024-11-07T15:25:22,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/833ff6abd57e43788ad7a26dbae4d882, entries=150, sequenceid=389, filesize=12.0 K 2024-11-07T15:25:22,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/e8f4e607f88c472a817f2367880914cb as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e8f4e607f88c472a817f2367880914cb 2024-11-07T15:25:22,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e8f4e607f88c472a817f2367880914cb, entries=150, sequenceid=389, filesize=12.0 K 2024-11-07T15:25:22,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5489735189eabe07c0ddbb2d15b5812f in 460ms, sequenceid=389, compaction requested=true 2024-11-07T15:25:22,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:22,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:25:22,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:22,803 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:22,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:25:22,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:22,803 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:22,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5489735189eabe07c0ddbb2d15b5812f:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:25:22,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:22,804 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:22,804 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 112001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:22,804 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/B is initiating minor compaction (all files) 2024-11-07T15:25:22,804 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/A is initiating minor compaction (all files) 2024-11-07T15:25:22,804 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/B in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:22,804 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/A in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:22,804 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/a3926ca7715b4750ac556862ff27c21a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/c2c84396fe264b3581092dd7af834644, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/833ff6abd57e43788ad7a26dbae4d882] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=36.9 K 2024-11-07T15:25:22,804 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/c5172b10fe994653bdd422bf28d1510f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/9adbff1eab2b4b999028685eb7f30190, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/378e6602c4514f919d3c0ac993f4fdf2] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=109.4 K 2024-11-07T15:25:22,804 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:22,804 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. files: [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/c5172b10fe994653bdd422bf28d1510f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/9adbff1eab2b4b999028685eb7f30190, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/378e6602c4514f919d3c0ac993f4fdf2] 2024-11-07T15:25:22,804 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting a3926ca7715b4750ac556862ff27c21a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1730993119381 2024-11-07T15:25:22,804 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5172b10fe994653bdd422bf28d1510f, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1730993119381 2024-11-07T15:25:22,805 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting c2c84396fe264b3581092dd7af834644, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730993120542 2024-11-07T15:25:22,805 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9adbff1eab2b4b999028685eb7f30190, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730993120542 2024-11-07T15:25:22,805 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 833ff6abd57e43788ad7a26dbae4d882, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1730993121710 2024-11-07T15:25:22,805 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] compactions.Compactor(225): Compacting 378e6602c4514f919d3c0ac993f4fdf2, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1730993121699 2024-11-07T15:25:22,812 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#B#compaction#520 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:22,812 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/5a79184d37fe4e649f711d770b4e3dee is 50, key is test_row_0/B:col10/1730993121710/Put/seqid=0 2024-11-07T15:25:22,833 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:22,851 INFO [Thread-2314 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2314. 2024-11-07T15:25:22,852 DEBUG [Thread-2314 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:25:22,852 DEBUG [Thread-2314 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:22,852 DEBUG [Thread-2314 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:22,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:25:22,853 INFO [Thread-2322 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2322. 2024-11-07T15:25:22,853 DEBUG [Thread-2322 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:25:22,853 DEBUG [Thread-2322 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:22,853 DEBUG [Thread-2322 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:22,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:25:22,853 INFO [Thread-2318 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2318. 2024-11-07T15:25:22,854 DEBUG [Thread-2318 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:25:22,854 DEBUG [Thread-2318 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:22,854 DEBUG [Thread-2318 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:22,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:25:22,855 INFO [Thread-2320 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2320. 2024-11-07T15:25:22,855 DEBUG [Thread-2320 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:25:22,855 DEBUG [Thread-2320 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:22,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742452_1628 (size=13255) 2024-11-07T15:25:22,855 DEBUG [Thread-2320 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:22,855 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:25:22,856 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107312bed6501f141e39434c2d6f0b2f659_5489735189eabe07c0ddbb2d15b5812f store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:22,856 INFO [Thread-2316 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2316. 2024-11-07T15:25:22,857 DEBUG [Thread-2316 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:25:22,857 DEBUG [Thread-2316 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:22,857 DEBUG [Thread-2316 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:22,857 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:25:22,859 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107312bed6501f141e39434c2d6f0b2f659_5489735189eabe07c0ddbb2d15b5812f, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:22,859 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107312bed6501f141e39434c2d6f0b2f659_5489735189eabe07c0ddbb2d15b5812f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:22,861 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/5a79184d37fe4e649f711d770b4e3dee as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/5a79184d37fe4e649f711d770b4e3dee 2024-11-07T15:25:22,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742453_1629 (size=4469) 2024-11-07T15:25:22,864 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#A#compaction#521 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:22,865 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/573967efcc9548d4a7e82133c59056a2 is 175, key is test_row_0/A:col10/1730993121710/Put/seqid=0 2024-11-07T15:25:22,868 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/B of 5489735189eabe07c0ddbb2d15b5812f into 5a79184d37fe4e649f711d770b4e3dee(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:22,868 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:22,868 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/B, priority=13, startTime=1730993122803; duration=0sec 2024-11-07T15:25:22,868 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:25:22,868 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:B 2024-11-07T15:25:22,868 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:25:22,869 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:25:22,869 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1541): 5489735189eabe07c0ddbb2d15b5812f/C is initiating minor compaction (all files) 2024-11-07T15:25:22,869 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5489735189eabe07c0ddbb2d15b5812f/C in TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:22,869 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/3b51a1d772cd43269664f6ca6b59b115, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/3fe38e7b43ef49f7a44f292bd921751f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e8f4e607f88c472a817f2367880914cb] into tmpdir=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp, totalSize=36.9 K 2024-11-07T15:25:22,870 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b51a1d772cd43269664f6ca6b59b115, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1730993119381 2024-11-07T15:25:22,870 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fe38e7b43ef49f7a44f292bd921751f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1730993120542 2024-11-07T15:25:22,870 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] compactions.Compactor(225): Compacting e8f4e607f88c472a817f2367880914cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1730993121710 2024-11-07T15:25:22,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742454_1630 (size=32209) 2024-11-07T15:25:22,877 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5489735189eabe07c0ddbb2d15b5812f#C#compaction#522 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:25:22,878 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/8900a0f6347b4856bb8ab0b9cf76cd45 is 50, key is test_row_0/C:col10/1730993121710/Put/seqid=0 2024-11-07T15:25:22,880 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/573967efcc9548d4a7e82133c59056a2 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/573967efcc9548d4a7e82133c59056a2 2024-11-07T15:25:22,885 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/A of 5489735189eabe07c0ddbb2d15b5812f into 573967efcc9548d4a7e82133c59056a2(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:22,885 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:22,885 INFO [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/A, priority=13, startTime=1730993122803; duration=0sec 2024-11-07T15:25:22,885 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:22,885 DEBUG [RS:0;7b3e4e8e12ef:33849-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:A 2024-11-07T15:25:22,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742455_1631 (size=13255) 2024-11-07T15:25:22,894 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/8900a0f6347b4856bb8ab0b9cf76cd45 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/8900a0f6347b4856bb8ab0b9cf76cd45 2024-11-07T15:25:22,899 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5489735189eabe07c0ddbb2d15b5812f/C of 5489735189eabe07c0ddbb2d15b5812f into 8900a0f6347b4856bb8ab0b9cf76cd45(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:25:22,899 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:22,899 INFO [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., storeName=5489735189eabe07c0ddbb2d15b5812f/C, priority=13, startTime=1730993122803; duration=0sec 2024-11-07T15:25:22,899 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:25:22,899 DEBUG [RS:0;7b3e4e8e12ef:33849-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5489735189eabe07c0ddbb2d15b5812f:C 2024-11-07T15:25:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33849 {}] regionserver.HRegion(8855): Flush requested on 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:23,039 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:25:23,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T15:25:23,039 INFO [Thread-2305 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2305. 2024-11-07T15:25:23,039 INFO [Thread-2309 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2309. 2024-11-07T15:25:23,039 DEBUG [Thread-2305 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:25:23,039 DEBUG [Thread-2309 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:25:23,039 DEBUG [Thread-2305 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:23,039 DEBUG [Thread-2309 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:23,039 DEBUG [Thread-2305 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:23,039 DEBUG [Thread-2309 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:23,039 INFO [Thread-2307 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2307. 2024-11-07T15:25:23,039 DEBUG [Thread-2307 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:25:23,039 DEBUG [Thread-2307 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:23,039 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:25:23,039 DEBUG [Thread-2307 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:23,039 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:25:23,039 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:25:23,043 INFO [Thread-2303 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2303. 2024-11-07T15:25:23,043 DEBUG [Thread-2303 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:25:23,043 DEBUG [Thread-2303 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:23,044 DEBUG [Thread-2303 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:23,044 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:25:23,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cd9c1566073e475eaed2424be57ee676_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_0/A:col10/1730993123039/Put/seqid=0 2024-11-07T15:25:23,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742456_1632 (size=12454) 2024-11-07T15:25:23,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-07T15:25:23,091 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:25:23,452 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:23,455 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cd9c1566073e475eaed2424be57ee676_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cd9c1566073e475eaed2424be57ee676_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:23,456 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/8d35474a8a684c2c90538d2dd3b8e717, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:23,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/8d35474a8a684c2c90538d2dd3b8e717 is 175, key is test_row_0/A:col10/1730993123039/Put/seqid=0 2024-11-07T15:25:23,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742457_1633 (size=31255) 2024-11-07T15:25:23,859 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/8d35474a8a684c2c90538d2dd3b8e717 2024-11-07T15:25:23,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/372b826c4e7344c7abef1591d4ed3e95 is 50, key is test_row_0/B:col10/1730993123039/Put/seqid=0 2024-11-07T15:25:23,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742458_1634 (size=12301) 2024-11-07T15:25:24,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/372b826c4e7344c7abef1591d4ed3e95 2024-11-07T15:25:24,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/8f5748f30f994e4baa6ced8d06b8e855 is 50, key is test_row_0/C:col10/1730993123039/Put/seqid=0 2024-11-07T15:25:24,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742459_1635 (size=12301) 2024-11-07T15:25:24,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/8f5748f30f994e4baa6ced8d06b8e855 2024-11-07T15:25:24,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/8d35474a8a684c2c90538d2dd3b8e717 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/8d35474a8a684c2c90538d2dd3b8e717 2024-11-07T15:25:24,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/8d35474a8a684c2c90538d2dd3b8e717, entries=150, sequenceid=413, filesize=30.5 K 2024-11-07T15:25:24,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/372b826c4e7344c7abef1591d4ed3e95 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/372b826c4e7344c7abef1591d4ed3e95 2024-11-07T15:25:24,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/372b826c4e7344c7abef1591d4ed3e95, entries=150, sequenceid=413, filesize=12.0 K 2024-11-07T15:25:24,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/8f5748f30f994e4baa6ced8d06b8e855 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/8f5748f30f994e4baa6ced8d06b8e855 2024-11-07T15:25:24,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/8f5748f30f994e4baa6ced8d06b8e855, entries=150, sequenceid=413, filesize=12.0 K 2024-11-07T15:25:24,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for 5489735189eabe07c0ddbb2d15b5812f in 1647ms, sequenceid=413, compaction requested=false 2024-11-07T15:25:24,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5489735189eabe07c0ddbb2d15b5812f: 2024-11-07T15:25:26,432 INFO [Thread-2311 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2311. 2024-11-07T15:25:26,432 DEBUG [Thread-2311 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:25:26,432 DEBUG [Thread-2311 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:26,433 DEBUG [Thread-2311 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:26,433 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:25:26,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T15:25:26,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-07T15:25:26,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-07T15:25:26,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-07T15:25:26,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-07T15:25:26,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-07T15:25:26,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T15:25:26,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3666 2024-11-07T15:25:26,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3601 2024-11-07T15:25:26,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3587 2024-11-07T15:25:26,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3694 2024-11-07T15:25:26,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3628 2024-11-07T15:25:26,433 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T15:25:26,433 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T15:25:26,433 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:25:26,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:26,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:26,433 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:25:26,434 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T15:25:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T15:25:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T15:25:26,436 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993126436"}]},"ts":"1730993126436"} 2024-11-07T15:25:26,437 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T15:25:26,437 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T15:25:26,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:25:26,439 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5489735189eabe07c0ddbb2d15b5812f, UNASSIGN}] 2024-11-07T15:25:26,439 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5489735189eabe07c0ddbb2d15b5812f, UNASSIGN 2024-11-07T15:25:26,440 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=5489735189eabe07c0ddbb2d15b5812f, regionState=CLOSING, regionLocation=7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:26,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5489735189eabe07c0ddbb2d15b5812f, UNASSIGN because future has completed 2024-11-07T15:25:26,441 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:25:26,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906}] 2024-11-07T15:25:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T15:25:26,593 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:26,593 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:25:26,593 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing 5489735189eabe07c0ddbb2d15b5812f, disabling compactions & flushes 2024-11-07T15:25:26,593 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:26,593 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:26,593 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. after waiting 0 ms 2024-11-07T15:25:26,593 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:26,593 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing 5489735189eabe07c0ddbb2d15b5812f 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-07T15:25:26,597 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f8484c366ff248d68001a5273f53f3df_5489735189eabe07c0ddbb2d15b5812f is 50, key is test_row_2/A:col10/1730993126431/Put/seqid=0 2024-11-07T15:25:26,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742460_1636 (size=7374) 2024-11-07T15:25:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T15:25:27,002 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:25:27,011 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f8484c366ff248d68001a5273f53f3df_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f8484c366ff248d68001a5273f53f3df_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:27,012 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/6d4d7fa8fad84afdba2bd4e06b0511ed, store: [table=TestAcidGuarantees family=A region=5489735189eabe07c0ddbb2d15b5812f] 2024-11-07T15:25:27,013 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/6d4d7fa8fad84afdba2bd4e06b0511ed is 175, key is test_row_2/A:col10/1730993126431/Put/seqid=0 2024-11-07T15:25:27,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742461_1637 (size=13865) 2024-11-07T15:25:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T15:25:27,418 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=417, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/6d4d7fa8fad84afdba2bd4e06b0511ed 2024-11-07T15:25:27,432 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/a2b46a45676148d08a165119ede2e65c is 50, key is test_row_2/B:col10/1730993126431/Put/seqid=0 2024-11-07T15:25:27,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742462_1638 (size=7415) 2024-11-07T15:25:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T15:25:27,837 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/a2b46a45676148d08a165119ede2e65c 2024-11-07T15:25:27,847 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/3fa4894608d842ab84814ed392c51dfe is 50, key is test_row_2/C:col10/1730993126431/Put/seqid=0 2024-11-07T15:25:27,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742463_1639 (size=7415) 2024-11-07T15:25:28,253 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/3fa4894608d842ab84814ed392c51dfe 2024-11-07T15:25:28,262 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/A/6d4d7fa8fad84afdba2bd4e06b0511ed as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/6d4d7fa8fad84afdba2bd4e06b0511ed 2024-11-07T15:25:28,268 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/6d4d7fa8fad84afdba2bd4e06b0511ed, entries=50, sequenceid=417, filesize=13.5 K 2024-11-07T15:25:28,269 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/B/a2b46a45676148d08a165119ede2e65c as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/a2b46a45676148d08a165119ede2e65c 2024-11-07T15:25:28,273 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/a2b46a45676148d08a165119ede2e65c, entries=50, sequenceid=417, filesize=7.2 K 2024-11-07T15:25:28,274 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/.tmp/C/3fa4894608d842ab84814ed392c51dfe as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/3fa4894608d842ab84814ed392c51dfe 2024-11-07T15:25:28,277 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/3fa4894608d842ab84814ed392c51dfe, entries=50, sequenceid=417, filesize=7.2 K 2024-11-07T15:25:28,278 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 5489735189eabe07c0ddbb2d15b5812f in 1685ms, sequenceid=417, compaction requested=true 2024-11-07T15:25:28,279 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/ea64d04f0cff42578c5ed9baacae4049, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/23d126cc8ea44d7cb41cb005c8eeef36, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/d9767f8f2b2f47d6be2cc4a1c34e6e9b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/882eba43c56146b1924a77093009aa76, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/f2fd0a83b4cf4191ac641b64db2b4f54, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/636160b5f6b548b88561ae2ee28f66d6, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/a88c44cbfaf040cda14ea331f1271c53, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/f7b9695efc9b42368c34532845067faf, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/8dde1d0b8d244968adb38e3c12332eba, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/b2f36fc5e838445aab180e6c2e558cae, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/fcd5a51d02ce47ab92016cff28499c2b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/3f4f26ca83794876a74f36884ca93468, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/4e9acc68bd9d45ed95a9e8faf3cbca78, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/6c1a7f27e6a047b3965f199bea48bc65, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/97f833ff0ee842f8b5d4726fb10e2daa, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/b65ccbbccd3e465586a930332442c251, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/d12cf0cf642548ea980629c598cda8e5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/c6a54e2301114e428e9a50851eb04908, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/2a5f9601bcac48a484e2a39cc5bd9625, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/e71e7a43178f4e0183bdca42da532963, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/e67fc72b74184bf9bfeb0ac0652bedb2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/7d1dd292dffc491f846a1ccdc95ca3d7, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/3361cd8c29404a69b0431f246e3867a8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/dbbc6aa31cee4fcd9ae16a2bd3f05106, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/c5172b10fe994653bdd422bf28d1510f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/761cff243da1437bb1c75c2528751f9f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/9adbff1eab2b4b999028685eb7f30190, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/378e6602c4514f919d3c0ac993f4fdf2] to archive 2024-11-07T15:25:28,279 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:25:28,281 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/ea64d04f0cff42578c5ed9baacae4049 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/ea64d04f0cff42578c5ed9baacae4049 2024-11-07T15:25:28,283 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/23d126cc8ea44d7cb41cb005c8eeef36 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/23d126cc8ea44d7cb41cb005c8eeef36 2024-11-07T15:25:28,284 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/d9767f8f2b2f47d6be2cc4a1c34e6e9b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/d9767f8f2b2f47d6be2cc4a1c34e6e9b 2024-11-07T15:25:28,286 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/882eba43c56146b1924a77093009aa76 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/882eba43c56146b1924a77093009aa76 2024-11-07T15:25:28,287 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/f2fd0a83b4cf4191ac641b64db2b4f54 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/f2fd0a83b4cf4191ac641b64db2b4f54 2024-11-07T15:25:28,288 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/636160b5f6b548b88561ae2ee28f66d6 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/636160b5f6b548b88561ae2ee28f66d6 2024-11-07T15:25:28,290 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/a88c44cbfaf040cda14ea331f1271c53 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/a88c44cbfaf040cda14ea331f1271c53 2024-11-07T15:25:28,291 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/f7b9695efc9b42368c34532845067faf to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/f7b9695efc9b42368c34532845067faf 2024-11-07T15:25:28,292 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/8dde1d0b8d244968adb38e3c12332eba to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/8dde1d0b8d244968adb38e3c12332eba 2024-11-07T15:25:28,293 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/b2f36fc5e838445aab180e6c2e558cae to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/b2f36fc5e838445aab180e6c2e558cae 2024-11-07T15:25:28,295 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/fcd5a51d02ce47ab92016cff28499c2b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/fcd5a51d02ce47ab92016cff28499c2b 2024-11-07T15:25:28,296 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/3f4f26ca83794876a74f36884ca93468 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/3f4f26ca83794876a74f36884ca93468 2024-11-07T15:25:28,298 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/4e9acc68bd9d45ed95a9e8faf3cbca78 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/4e9acc68bd9d45ed95a9e8faf3cbca78 2024-11-07T15:25:28,299 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/6c1a7f27e6a047b3965f199bea48bc65 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/6c1a7f27e6a047b3965f199bea48bc65 2024-11-07T15:25:28,301 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/97f833ff0ee842f8b5d4726fb10e2daa to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/97f833ff0ee842f8b5d4726fb10e2daa 2024-11-07T15:25:28,302 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/b65ccbbccd3e465586a930332442c251 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/b65ccbbccd3e465586a930332442c251 2024-11-07T15:25:28,304 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/d12cf0cf642548ea980629c598cda8e5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/d12cf0cf642548ea980629c598cda8e5 2024-11-07T15:25:28,305 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/c6a54e2301114e428e9a50851eb04908 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/c6a54e2301114e428e9a50851eb04908 2024-11-07T15:25:28,306 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/2a5f9601bcac48a484e2a39cc5bd9625 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/2a5f9601bcac48a484e2a39cc5bd9625 2024-11-07T15:25:28,308 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/e71e7a43178f4e0183bdca42da532963 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/e71e7a43178f4e0183bdca42da532963 2024-11-07T15:25:28,309 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/e67fc72b74184bf9bfeb0ac0652bedb2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/e67fc72b74184bf9bfeb0ac0652bedb2 2024-11-07T15:25:28,310 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/7d1dd292dffc491f846a1ccdc95ca3d7 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/7d1dd292dffc491f846a1ccdc95ca3d7 2024-11-07T15:25:28,311 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/3361cd8c29404a69b0431f246e3867a8 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/3361cd8c29404a69b0431f246e3867a8 2024-11-07T15:25:28,312 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/dbbc6aa31cee4fcd9ae16a2bd3f05106 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/dbbc6aa31cee4fcd9ae16a2bd3f05106 2024-11-07T15:25:28,313 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/c5172b10fe994653bdd422bf28d1510f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/c5172b10fe994653bdd422bf28d1510f 2024-11-07T15:25:28,314 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/761cff243da1437bb1c75c2528751f9f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/761cff243da1437bb1c75c2528751f9f 2024-11-07T15:25:28,315 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/9adbff1eab2b4b999028685eb7f30190 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/9adbff1eab2b4b999028685eb7f30190 2024-11-07T15:25:28,316 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/378e6602c4514f919d3c0ac993f4fdf2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/378e6602c4514f919d3c0ac993f4fdf2 2024-11-07T15:25:28,317 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/563cf0201ce34071a3c85d1dbfa27033, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/03582e749d014b20ad2155802d07f592, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/790e28c55eaa41f58b726b03b436ba1e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/3b9de8068ec7488eab40c701696fca4f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/dc7ce05bf9784a87851c18ba0aef1ad3, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/75825efcbb844b7986dff42a064d5522, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/4b3d31f844f4479c96ba0ec39efd66a1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/f8c85e587aee471289d6573077192738, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/75d4dd459e734c5ea83efb4b4c495fea, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/9b3b99e44def4779a77e45255851f661, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/7826bbaea98745d9b77e6b0f014c6e38, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/bce0a27df8a34f2f9340aa963bc21a38, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/778f374e50ae4a538b2df1fcd09a981e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/874f9af43b394fdea8dac63ac82077d8, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/224478a57d8043cfa33fc9d79321eca2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/5447daa11855461087ddfde917bec5e5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/28a63483c4b64120b33c376001156a5f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/844c1ec7e2ae47cbaa3be1e8d2045ae1, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/0473aa3a6132493e9e0897e1e9aad1c2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/f9e120d0e5ff4380afe70dbf4925cf05, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/38f4ffc5cf3b4bbab399c802ce5f832d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/0cb6390362ea4017be3530dd547d3b76, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/a45a5915171b48e3aae541af71e590c2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/6da4a90563a04153a09d31377f903588, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/a3926ca7715b4750ac556862ff27c21a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/ec4ad6e9a157404792f477161cf59fef, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/c2c84396fe264b3581092dd7af834644, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/833ff6abd57e43788ad7a26dbae4d882] to archive 2024-11-07T15:25:28,318 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:25:28,319 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/563cf0201ce34071a3c85d1dbfa27033 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/563cf0201ce34071a3c85d1dbfa27033 2024-11-07T15:25:28,320 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/03582e749d014b20ad2155802d07f592 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/03582e749d014b20ad2155802d07f592 2024-11-07T15:25:28,321 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/790e28c55eaa41f58b726b03b436ba1e to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/790e28c55eaa41f58b726b03b436ba1e 2024-11-07T15:25:28,322 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/3b9de8068ec7488eab40c701696fca4f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/3b9de8068ec7488eab40c701696fca4f 2024-11-07T15:25:28,323 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/dc7ce05bf9784a87851c18ba0aef1ad3 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/dc7ce05bf9784a87851c18ba0aef1ad3 2024-11-07T15:25:28,324 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/75825efcbb844b7986dff42a064d5522 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/75825efcbb844b7986dff42a064d5522 2024-11-07T15:25:28,325 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/4b3d31f844f4479c96ba0ec39efd66a1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/4b3d31f844f4479c96ba0ec39efd66a1 2024-11-07T15:25:28,326 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/f8c85e587aee471289d6573077192738 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/f8c85e587aee471289d6573077192738 2024-11-07T15:25:28,327 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/75d4dd459e734c5ea83efb4b4c495fea to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/75d4dd459e734c5ea83efb4b4c495fea 2024-11-07T15:25:28,328 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/9b3b99e44def4779a77e45255851f661 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/9b3b99e44def4779a77e45255851f661 2024-11-07T15:25:28,329 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/7826bbaea98745d9b77e6b0f014c6e38 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/7826bbaea98745d9b77e6b0f014c6e38 2024-11-07T15:25:28,330 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/bce0a27df8a34f2f9340aa963bc21a38 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/bce0a27df8a34f2f9340aa963bc21a38 2024-11-07T15:25:28,331 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/778f374e50ae4a538b2df1fcd09a981e to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/778f374e50ae4a538b2df1fcd09a981e 2024-11-07T15:25:28,332 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/874f9af43b394fdea8dac63ac82077d8 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/874f9af43b394fdea8dac63ac82077d8 2024-11-07T15:25:28,333 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/224478a57d8043cfa33fc9d79321eca2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/224478a57d8043cfa33fc9d79321eca2 2024-11-07T15:25:28,334 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/5447daa11855461087ddfde917bec5e5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/5447daa11855461087ddfde917bec5e5 2024-11-07T15:25:28,335 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/28a63483c4b64120b33c376001156a5f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/28a63483c4b64120b33c376001156a5f 2024-11-07T15:25:28,336 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/844c1ec7e2ae47cbaa3be1e8d2045ae1 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/844c1ec7e2ae47cbaa3be1e8d2045ae1 2024-11-07T15:25:28,337 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/0473aa3a6132493e9e0897e1e9aad1c2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/0473aa3a6132493e9e0897e1e9aad1c2 2024-11-07T15:25:28,338 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/f9e120d0e5ff4380afe70dbf4925cf05 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/f9e120d0e5ff4380afe70dbf4925cf05 2024-11-07T15:25:28,340 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/38f4ffc5cf3b4bbab399c802ce5f832d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/38f4ffc5cf3b4bbab399c802ce5f832d 2024-11-07T15:25:28,341 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/0cb6390362ea4017be3530dd547d3b76 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/0cb6390362ea4017be3530dd547d3b76 2024-11-07T15:25:28,342 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/a45a5915171b48e3aae541af71e590c2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/a45a5915171b48e3aae541af71e590c2 2024-11-07T15:25:28,343 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/6da4a90563a04153a09d31377f903588 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/6da4a90563a04153a09d31377f903588 2024-11-07T15:25:28,344 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/a3926ca7715b4750ac556862ff27c21a to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/a3926ca7715b4750ac556862ff27c21a 2024-11-07T15:25:28,345 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/ec4ad6e9a157404792f477161cf59fef to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/ec4ad6e9a157404792f477161cf59fef 2024-11-07T15:25:28,346 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/c2c84396fe264b3581092dd7af834644 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/c2c84396fe264b3581092dd7af834644 2024-11-07T15:25:28,347 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/833ff6abd57e43788ad7a26dbae4d882 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/833ff6abd57e43788ad7a26dbae4d882 2024-11-07T15:25:28,349 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/2a5f683b40d34cf688f5ca817c36012b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/0f433ef42b434e649518b444a898035c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/4b60647ce54243bf88d5bd77d9f90613, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/6f85c8dc164d4cf4a90ae16b926ec54b, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/755868d5f90c44629900330b2f6e3635, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e3527c7ab45d435a9ed3ffe64b379710, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e95222c107f648989ddc1d0b58f67e36, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1113ddd2b7bf47b2bc7c381cc98fae5c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/385397b46fc04a429457906aea48f355, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/aef9f8bebb704b92959d429bfdd9d498, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1d8b1abdbf30481cbfd5903bd0b200a4, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/24766d8c28c24fd2913ee5f31752192c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/75880788678b4d538174702f437d265e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/c9b3c2fd7a45403c887198932b308e8d, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/bc7e8a0468a4422eb2e4075f2b09d772, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/773d9324d5d94fcc9ad637851cfb7687, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/7893eb5c0d0f4ebf8c41a5969c0c4e32, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/071cba4525cc4945b6f6118709fd414e, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/9610029e87e644519583b03092327bc5, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/d8bf8b10419b4c85b4a2fc8c0edc9056, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/f2d8e811235b4f179df896e61dbf7bd2, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/fb8ddb316d914096bff63cce566438b7, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1d6331bdca6e41d2bed25424c085932a, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/167e7da5d8604a31b88acc5084ca4a7c, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/3b51a1d772cd43269664f6ca6b59b115, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/259e5d3242bf4706afb8937c7c343efa, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/3fe38e7b43ef49f7a44f292bd921751f, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e8f4e607f88c472a817f2367880914cb] to archive 2024-11-07T15:25:28,349 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:25:28,351 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/2a5f683b40d34cf688f5ca817c36012b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/2a5f683b40d34cf688f5ca817c36012b 2024-11-07T15:25:28,352 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/0f433ef42b434e649518b444a898035c to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/0f433ef42b434e649518b444a898035c 2024-11-07T15:25:28,353 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/4b60647ce54243bf88d5bd77d9f90613 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/4b60647ce54243bf88d5bd77d9f90613 2024-11-07T15:25:28,354 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/6f85c8dc164d4cf4a90ae16b926ec54b to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/6f85c8dc164d4cf4a90ae16b926ec54b 2024-11-07T15:25:28,355 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/755868d5f90c44629900330b2f6e3635 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/755868d5f90c44629900330b2f6e3635 2024-11-07T15:25:28,356 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e3527c7ab45d435a9ed3ffe64b379710 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e3527c7ab45d435a9ed3ffe64b379710 2024-11-07T15:25:28,357 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e95222c107f648989ddc1d0b58f67e36 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e95222c107f648989ddc1d0b58f67e36 2024-11-07T15:25:28,358 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1113ddd2b7bf47b2bc7c381cc98fae5c to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1113ddd2b7bf47b2bc7c381cc98fae5c 2024-11-07T15:25:28,359 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/385397b46fc04a429457906aea48f355 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/385397b46fc04a429457906aea48f355 2024-11-07T15:25:28,360 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/aef9f8bebb704b92959d429bfdd9d498 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/aef9f8bebb704b92959d429bfdd9d498 2024-11-07T15:25:28,362 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1d8b1abdbf30481cbfd5903bd0b200a4 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1d8b1abdbf30481cbfd5903bd0b200a4 2024-11-07T15:25:28,363 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/24766d8c28c24fd2913ee5f31752192c to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/24766d8c28c24fd2913ee5f31752192c 2024-11-07T15:25:28,364 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/75880788678b4d538174702f437d265e to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/75880788678b4d538174702f437d265e 2024-11-07T15:25:28,365 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/c9b3c2fd7a45403c887198932b308e8d to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/c9b3c2fd7a45403c887198932b308e8d 2024-11-07T15:25:28,366 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/bc7e8a0468a4422eb2e4075f2b09d772 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/bc7e8a0468a4422eb2e4075f2b09d772 2024-11-07T15:25:28,367 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/773d9324d5d94fcc9ad637851cfb7687 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/773d9324d5d94fcc9ad637851cfb7687 2024-11-07T15:25:28,368 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/7893eb5c0d0f4ebf8c41a5969c0c4e32 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/7893eb5c0d0f4ebf8c41a5969c0c4e32 2024-11-07T15:25:28,369 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/071cba4525cc4945b6f6118709fd414e to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/071cba4525cc4945b6f6118709fd414e 2024-11-07T15:25:28,371 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/9610029e87e644519583b03092327bc5 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/9610029e87e644519583b03092327bc5 2024-11-07T15:25:28,372 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/d8bf8b10419b4c85b4a2fc8c0edc9056 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/d8bf8b10419b4c85b4a2fc8c0edc9056 2024-11-07T15:25:28,373 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/f2d8e811235b4f179df896e61dbf7bd2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/f2d8e811235b4f179df896e61dbf7bd2 2024-11-07T15:25:28,374 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/fb8ddb316d914096bff63cce566438b7 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/fb8ddb316d914096bff63cce566438b7 2024-11-07T15:25:28,375 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1d6331bdca6e41d2bed25424c085932a to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/1d6331bdca6e41d2bed25424c085932a 2024-11-07T15:25:28,376 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/167e7da5d8604a31b88acc5084ca4a7c to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/167e7da5d8604a31b88acc5084ca4a7c 2024-11-07T15:25:28,377 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/3b51a1d772cd43269664f6ca6b59b115 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/3b51a1d772cd43269664f6ca6b59b115 2024-11-07T15:25:28,378 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/259e5d3242bf4706afb8937c7c343efa to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/259e5d3242bf4706afb8937c7c343efa 2024-11-07T15:25:28,379 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/3fe38e7b43ef49f7a44f292bd921751f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/3fe38e7b43ef49f7a44f292bd921751f 2024-11-07T15:25:28,380 DEBUG [StoreCloser-TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e8f4e607f88c472a817f2367880914cb to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/e8f4e607f88c472a817f2367880914cb 2024-11-07T15:25:28,385 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/recovered.edits/420.seqid, newMaxSeqId=420, maxSeqId=4 2024-11-07T15:25:28,386 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. 2024-11-07T15:25:28,386 DEBUG [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for 5489735189eabe07c0ddbb2d15b5812f: Waiting for close lock at 1730993126593Running coprocessor pre-close hooks at 1730993126593Disabling compacts and flushes for region at 1730993126593Disabling writes for close at 1730993126593Obtaining lock to block concurrent updates at 1730993126593Preparing flush snapshotting stores in 5489735189eabe07c0ddbb2d15b5812f at 1730993126593Finished memstore snapshotting TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1730993126593Flushing stores of TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f. at 1730993126594 (+1 ms)Flushing 5489735189eabe07c0ddbb2d15b5812f/A: creating writer at 1730993126594Flushing mob file 5489735189eabe07c0ddbb2d15b5812f/A: appending metadata at 1730993126597 (+3 ms)Flushing mob file 5489735189eabe07c0ddbb2d15b5812f/A: closing flushed file at 1730993126597Flushing 5489735189eabe07c0ddbb2d15b5812f/A: appending metadata at 1730993127012 (+415 ms)Flushing 5489735189eabe07c0ddbb2d15b5812f/A: closing flushed file at 1730993127012Flushing 5489735189eabe07c0ddbb2d15b5812f/B: creating writer at 1730993127427 (+415 ms)Flushing 5489735189eabe07c0ddbb2d15b5812f/B: appending metadata at 1730993127430 (+3 ms)Flushing 5489735189eabe07c0ddbb2d15b5812f/B: closing flushed file at 1730993127431 (+1 ms)Flushing 5489735189eabe07c0ddbb2d15b5812f/C: creating writer at 1730993127844 (+413 ms)Flushing 5489735189eabe07c0ddbb2d15b5812f/C: appending metadata at 1730993127847 (+3 ms)Flushing 5489735189eabe07c0ddbb2d15b5812f/C: closing flushed file at 1730993127847Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@157936b6: reopening flushed file at 1730993128260 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ea5ba5: reopening flushed file at 1730993128268 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@404baf8f: reopening flushed file at 1730993128273 (+5 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 5489735189eabe07c0ddbb2d15b5812f in 1685ms, sequenceid=417, compaction requested=true at 1730993128278 (+5 ms)Writing region close event to WAL at 1730993128382 (+104 ms)Running coprocessor post-close hooks at 1730993128386 (+4 ms)Closed at 1730993128386 2024-11-07T15:25:28,388 INFO [RS_CLOSE_REGION-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed 5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,389 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=5489735189eabe07c0ddbb2d15b5812f, regionState=CLOSED 2024-11-07T15:25:28,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 because future has completed 2024-11-07T15:25:28,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-07T15:25:28,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure 5489735189eabe07c0ddbb2d15b5812f, server=7b3e4e8e12ef,33849,1730992951906 in 1.9500 sec 2024-11-07T15:25:28,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-11-07T15:25:28,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5489735189eabe07c0ddbb2d15b5812f, UNASSIGN in 1.9550 sec 2024-11-07T15:25:28,400 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-07T15:25:28,400 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9590 sec 2024-11-07T15:25:28,402 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730993128402"}]},"ts":"1730993128402"} 2024-11-07T15:25:28,404 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T15:25:28,404 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T15:25:28,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9710 sec 2024-11-07T15:25:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T15:25:28,582 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:25:28,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T15:25:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:25:28,588 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:25:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T15:25:28,590 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:25:28,595 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,599 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C, FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/recovered.edits] 2024-11-07T15:25:28,604 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/573967efcc9548d4a7e82133c59056a2 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/573967efcc9548d4a7e82133c59056a2 2024-11-07T15:25:28,607 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/6d4d7fa8fad84afdba2bd4e06b0511ed to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/6d4d7fa8fad84afdba2bd4e06b0511ed 2024-11-07T15:25:28,609 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/8d35474a8a684c2c90538d2dd3b8e717 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/A/8d35474a8a684c2c90538d2dd3b8e717 2024-11-07T15:25:28,613 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/372b826c4e7344c7abef1591d4ed3e95 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/372b826c4e7344c7abef1591d4ed3e95 2024-11-07T15:25:28,614 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/5a79184d37fe4e649f711d770b4e3dee to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/5a79184d37fe4e649f711d770b4e3dee 2024-11-07T15:25:28,616 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/a2b46a45676148d08a165119ede2e65c to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/B/a2b46a45676148d08a165119ede2e65c 2024-11-07T15:25:28,620 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/3fa4894608d842ab84814ed392c51dfe to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/3fa4894608d842ab84814ed392c51dfe 2024-11-07T15:25:28,622 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/8900a0f6347b4856bb8ab0b9cf76cd45 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/8900a0f6347b4856bb8ab0b9cf76cd45 2024-11-07T15:25:28,624 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/8f5748f30f994e4baa6ced8d06b8e855 to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/C/8f5748f30f994e4baa6ced8d06b8e855 2024-11-07T15:25:28,629 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/recovered.edits/420.seqid to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f/recovered.edits/420.seqid 2024-11-07T15:25:28,630 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/default/TestAcidGuarantees/5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,630 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T15:25:28,631 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T15:25:28,632 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T15:25:28,638 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070acae8bd65a446798c97fedd01cea9d6_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070acae8bd65a446798c97fedd01cea9d6_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,640 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110721b3b4d09cd64073a91412f65c6ee1f8_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110721b3b4d09cd64073a91412f65c6ee1f8_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,642 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072e89a0907b2945f987a25adf366746fd_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072e89a0907b2945f987a25adf366746fd_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,643 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075323fe16c57c48a69516e89f5ae360ba_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075323fe16c57c48a69516e89f5ae360ba_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,645 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107616da3650db6495fb15c63c946a9b1b5_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107616da3650db6495fb15c63c946a9b1b5_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,646 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110765766928d41f4892a79f11bd7b9856b5_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110765766928d41f4892a79f11bd7b9856b5_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,648 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107707563a2f93a4aadb46bb5366270f600_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107707563a2f93a4aadb46bb5366270f600_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,649 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107803604eb22034918a9f5b6dc9aaf859c_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107803604eb22034918a9f5b6dc9aaf859c_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,650 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078245940a5a6e45c991139f55b5c4d552_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078245940a5a6e45c991139f55b5c4d552_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,652 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110794308e8e53d14d16af0d0d40ac279ebc_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110794308e8e53d14d16af0d0d40ac279ebc_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,653 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110796dc29f457f44c2fa2f9bb2922facfc3_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110796dc29f457f44c2fa2f9bb2922facfc3_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,655 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079be77fea6a984aab80604239173c2f6f_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079be77fea6a984aab80604239173c2f6f_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,657 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a8828c673ae040a6a5b50cb87af11c1e_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a8828c673ae040a6a5b50cb87af11c1e_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,659 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b0da33757a0c413ba1c393fdb99fc642_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b0da33757a0c413ba1c393fdb99fc642_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,660 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b7239d82b6a6486ebeb9b88f6dc3f896_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b7239d82b6a6486ebeb9b88f6dc3f896_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,662 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bc0fcce2f4c540548afb6b0f2a4f9ffe_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bc0fcce2f4c540548afb6b0f2a4f9ffe_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,664 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bca590a482c1477fa28a10b81a18159f_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bca590a482c1477fa28a10b81a18159f_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,665 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c4cbf896f8514f0ba5b9b5a712b41646_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c4cbf896f8514f0ba5b9b5a712b41646_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,667 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ca825e986a5c46138150b79214c5263f_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ca825e986a5c46138150b79214c5263f_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,669 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cd9c1566073e475eaed2424be57ee676_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cd9c1566073e475eaed2424be57ee676_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,671 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ed49d9e836c446d596ac0e6eac3b9106_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107ed49d9e836c446d596ac0e6eac3b9106_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,673 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f8484c366ff248d68001a5273f53f3df_5489735189eabe07c0ddbb2d15b5812f to hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f8484c366ff248d68001a5273f53f3df_5489735189eabe07c0ddbb2d15b5812f 2024-11-07T15:25:28,673 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T15:25:28,676 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:25:28,679 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T15:25:28,682 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T15:25:28,683 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:25:28,683 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T15:25:28,684 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730993128683"}]},"ts":"9223372036854775807"} 2024-11-07T15:25:28,686 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T15:25:28,686 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5489735189eabe07c0ddbb2d15b5812f, NAME => 'TestAcidGuarantees,,1730993099758.5489735189eabe07c0ddbb2d15b5812f.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T15:25:28,686 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T15:25:28,686 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730993128686"}]},"ts":"9223372036854775807"} 2024-11-07T15:25:28,688 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T15:25:28,689 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:25:28,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 105 msec 2024-11-07T15:25:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46811 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-07T15:25:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T15:25:28,705 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:25:28,716 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242 (was 240) - Thread LEAK? -, OpenFileDescriptor=457 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 395) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1133 (was 1223) 2024-11-07T15:25:28,716 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-07T15:25:28,716 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T15:25:28,716 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:25:28,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:28,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:28,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:25:28,717 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T15:25:28,717 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1258311727, stopped=false 2024-11-07T15:25:28,717 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7b3e4e8e12ef,46811,1730992951165 2024-11-07T15:25:28,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T15:25:28,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T15:25:28,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:25:28,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:25:28,796 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T15:25:28,797 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T15:25:28,797 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:25:28,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:28,797 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T15:25:28,798 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T15:25:28,798 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7b3e4e8e12ef,33849,1730992951906' ***** 2024-11-07T15:25:28,798 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T15:25:28,799 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T15:25:28,799 INFO [RS:0;7b3e4e8e12ef:33849 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T15:25:28,799 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T15:25:28,800 INFO [RS:0;7b3e4e8e12ef:33849 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T15:25:28,800 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(959): stopping server 7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:28,800 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T15:25:28,800 INFO [RS:0;7b3e4e8e12ef:33849 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7b3e4e8e12ef:33849. 2024-11-07T15:25:28,800 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:25:28,800 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:28,800 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T15:25:28,801 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T15:25:28,801 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T15:25:28,801 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-07T15:25:28,802 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-07T15:25:28,802 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-07T15:25:28,802 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:25:28,802 DEBUG [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T15:25:28,802 INFO [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T15:25:28,802 DEBUG [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T15:25:28,802 DEBUG [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T15:25:28,802 DEBUG [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T15:25:28,802 INFO [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-07T15:25:28,824 DEBUG [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/.tmp/info/28e630341ceb4fed872b1fe5eaead155 is 95, key is TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601./info:/1730992983767/DeleteFamily/seqid=0 2024-11-07T15:25:28,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742464_1640 (size=6514) 2024-11-07T15:25:28,860 INFO [regionserver/7b3e4e8e12ef:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T15:25:29,002 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:25:29,203 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:25:29,229 INFO [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/.tmp/info/28e630341ceb4fed872b1fe5eaead155 2024-11-07T15:25:29,255 DEBUG [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/.tmp/ns/d4e81fea965a4edc8995a25da2ffd728 is 93, key is TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601./ns:/1730992983732/DeleteFamily/seqid=0 2024-11-07T15:25:29,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742465_1641 (size=5906) 2024-11-07T15:25:29,345 INFO [regionserver/7b3e4e8e12ef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-07T15:25:29,345 INFO [regionserver/7b3e4e8e12ef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-07T15:25:29,403 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:25:29,603 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:25:29,660 INFO [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/.tmp/ns/d4e81fea965a4edc8995a25da2ffd728 2024-11-07T15:25:29,687 DEBUG [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/.tmp/rep_barrier/b57c5390da6742c596c36bab8cdfc26e is 102, key is TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601./rep_barrier:/1730992983732/DeleteFamily/seqid=0 2024-11-07T15:25:29,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742466_1642 (size=6025) 2024-11-07T15:25:29,803 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:25:30,004 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:25:30,095 INFO [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/.tmp/rep_barrier/b57c5390da6742c596c36bab8cdfc26e 2024-11-07T15:25:30,120 DEBUG [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/.tmp/table/01b77e0c23cf4f87b7fcd0847125c7ee is 96, key is TestAcidGuarantees,,1730992956121.86b0da056c7c770742d9ba46450cf601./table:/1730992983732/DeleteFamily/seqid=0 2024-11-07T15:25:30,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742467_1643 (size=5936) 2024-11-07T15:25:30,204 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:25:30,294 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T15:25:30,404 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:25:30,525 INFO [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/.tmp/table/01b77e0c23cf4f87b7fcd0847125c7ee 2024-11-07T15:25:30,535 DEBUG [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/.tmp/info/28e630341ceb4fed872b1fe5eaead155 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/info/28e630341ceb4fed872b1fe5eaead155 2024-11-07T15:25:30,541 INFO [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/info/28e630341ceb4fed872b1fe5eaead155, entries=12, sequenceid=90, filesize=6.4 K 2024-11-07T15:25:30,542 DEBUG [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/.tmp/ns/d4e81fea965a4edc8995a25da2ffd728 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/ns/d4e81fea965a4edc8995a25da2ffd728 2024-11-07T15:25:30,546 INFO [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/ns/d4e81fea965a4edc8995a25da2ffd728, entries=8, sequenceid=90, filesize=5.8 K 2024-11-07T15:25:30,546 DEBUG [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/.tmp/rep_barrier/b57c5390da6742c596c36bab8cdfc26e as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/rep_barrier/b57c5390da6742c596c36bab8cdfc26e 2024-11-07T15:25:30,550 INFO [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/rep_barrier/b57c5390da6742c596c36bab8cdfc26e, entries=6, sequenceid=90, filesize=5.9 K 2024-11-07T15:25:30,550 DEBUG [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/.tmp/table/01b77e0c23cf4f87b7fcd0847125c7ee as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/table/01b77e0c23cf4f87b7fcd0847125c7ee 2024-11-07T15:25:30,553 INFO [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/table/01b77e0c23cf4f87b7fcd0847125c7ee, entries=7, sequenceid=90, filesize=5.8 K 2024-11-07T15:25:30,553 INFO [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1751ms, sequenceid=90, compaction requested=false 2024-11-07T15:25:30,557 DEBUG [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-07T15:25:30,557 DEBUG [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T15:25:30,557 INFO [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T15:25:30,557 DEBUG [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730993128802Running coprocessor pre-close hooks at 1730993128802Disabling compacts and flushes for region at 1730993128802Disabling writes for close at 1730993128802Obtaining lock to block concurrent updates at 1730993128802Preparing flush snapshotting stores in 1588230740 at 1730993128802Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1730993128803 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1730993128803Flushing 1588230740/info: creating writer at 1730993128804 (+1 ms)Flushing 1588230740/info: appending metadata at 1730993128823 (+19 ms)Flushing 1588230740/info: closing flushed file at 1730993128823Flushing 1588230740/ns: creating writer at 1730993129237 (+414 ms)Flushing 1588230740/ns: appending metadata at 1730993129255 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1730993129255Flushing 1588230740/rep_barrier: creating writer at 1730993129670 (+415 ms)Flushing 1588230740/rep_barrier: appending metadata at 1730993129687 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1730993129687Flushing 1588230740/table: creating writer at 1730993130104 (+417 ms)Flushing 1588230740/table: appending metadata at 1730993130120 (+16 ms)Flushing 1588230740/table: closing flushed file at 1730993130120Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@671f3cb4: reopening flushed file at 1730993130534 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11966913: reopening flushed file at 1730993130541 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71fcf07a: reopening flushed file at 1730993130546 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47718c39: reopening flushed file at 1730993130550 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1751ms, sequenceid=90, compaction requested=false at 1730993130553 (+3 ms)Writing region close event to WAL at 1730993130555 (+2 ms)Running coprocessor post-close hooks at 1730993130557 (+2 ms)Closed at 1730993130557 2024-11-07T15:25:30,558 DEBUG [RS_CLOSE_META-regionserver/7b3e4e8e12ef:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T15:25:30,605 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(976): stopping server 7b3e4e8e12ef,33849,1730992951906; all regions closed. 2024-11-07T15:25:30,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741834_1010 (size=25344) 2024-11-07T15:25:30,613 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/oldWALs 2024-11-07T15:25:30,613 INFO [RS:0;7b3e4e8e12ef:33849 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7b3e4e8e12ef%2C33849%2C1730992951906.meta:.meta(num 1730992955629) 2024-11-07T15:25:30,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741832_1008 (size=14548080) 2024-11-07T15:25:30,619 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/oldWALs 2024-11-07T15:25:30,619 INFO [RS:0;7b3e4e8e12ef:33849 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7b3e4e8e12ef%2C33849%2C1730992951906:(num 1730992954510) 2024-11-07T15:25:30,619 DEBUG [RS:0;7b3e4e8e12ef:33849 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:25:30,619 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T15:25:30,619 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T15:25:30,620 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.ChoreService(370): Chore service for: regionserver/7b3e4e8e12ef:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-07T15:25:30,620 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T15:25:30,620 INFO [regionserver/7b3e4e8e12ef:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T15:25:30,621 INFO [RS:0;7b3e4e8e12ef:33849 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33849 2024-11-07T15:25:30,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7b3e4e8e12ef,33849,1730992951906 2024-11-07T15:25:30,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T15:25:30,680 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T15:25:30,692 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7b3e4e8e12ef,33849,1730992951906] 2024-11-07T15:25:30,702 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7b3e4e8e12ef,33849,1730992951906 already deleted, retry=false 2024-11-07T15:25:30,702 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7b3e4e8e12ef,33849,1730992951906 expired; onlineServers=0 2024-11-07T15:25:30,702 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7b3e4e8e12ef,46811,1730992951165' ***** 2024-11-07T15:25:30,702 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T15:25:30,703 INFO [M:0;7b3e4e8e12ef:46811 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T15:25:30,703 INFO [M:0;7b3e4e8e12ef:46811 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T15:25:30,703 DEBUG [M:0;7b3e4e8e12ef:46811 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T15:25:30,703 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T15:25:30,703 DEBUG [M:0;7b3e4e8e12ef:46811 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T15:25:30,703 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster-HFileCleaner.small.0-1730992954233 {}] cleaner.HFileCleaner(306): Exit Thread[master/7b3e4e8e12ef:0:becomeActiveMaster-HFileCleaner.small.0-1730992954233,5,FailOnTimeoutGroup] 2024-11-07T15:25:30,703 INFO [M:0;7b3e4e8e12ef:46811 {}] hbase.ChoreService(370): Chore service for: master/7b3e4e8e12ef:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-07T15:25:30,703 INFO [M:0;7b3e4e8e12ef:46811 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T15:25:30,703 DEBUG [M:0;7b3e4e8e12ef:46811 {}] master.HMaster(1795): Stopping service threads 2024-11-07T15:25:30,703 INFO [M:0;7b3e4e8e12ef:46811 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T15:25:30,703 DEBUG [master/7b3e4e8e12ef:0:becomeActiveMaster-HFileCleaner.large.0-1730992954229 {}] cleaner.HFileCleaner(306): Exit Thread[master/7b3e4e8e12ef:0:becomeActiveMaster-HFileCleaner.large.0-1730992954229,5,FailOnTimeoutGroup] 2024-11-07T15:25:30,704 INFO [M:0;7b3e4e8e12ef:46811 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T15:25:30,704 ERROR [M:0;7b3e4e8e12ef:46811 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (726517346) connection to localhost/127.0.0.1:41161 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:41161,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-07T15:25:30,704 INFO [M:0;7b3e4e8e12ef:46811 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T15:25:30,705 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T15:25:30,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T15:25:30,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:25:30,713 DEBUG [M:0;7b3e4e8e12ef:46811 {}] zookeeper.ZKUtil(347): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-07T15:25:30,713 WARN [M:0;7b3e4e8e12ef:46811 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-07T15:25:30,714 INFO [M:0;7b3e4e8e12ef:46811 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/.lastflushedseqids 2024-11-07T15:25:30,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742468_1644 (size=72) 2024-11-07T15:25:30,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T15:25:30,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33849-0x10115e447f30001, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T15:25:30,793 INFO [RS:0;7b3e4e8e12ef:33849 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T15:25:30,793 INFO [RS:0;7b3e4e8e12ef:33849 {}] regionserver.HRegionServer(1031): Exiting; stopping=7b3e4e8e12ef,33849,1730992951906; zookeeper connection closed. 2024-11-07T15:25:30,794 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@dd833a5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@dd833a5 2024-11-07T15:25:30,795 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-07T15:25:31,130 INFO [M:0;7b3e4e8e12ef:46811 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T15:25:31,130 INFO [M:0;7b3e4e8e12ef:46811 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T15:25:31,130 DEBUG [M:0;7b3e4e8e12ef:46811 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T15:25:31,131 INFO [M:0;7b3e4e8e12ef:46811 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T15:25:31,131 DEBUG [M:0;7b3e4e8e12ef:46811 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T15:25:31,131 DEBUG [M:0;7b3e4e8e12ef:46811 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T15:25:31,131 DEBUG [M:0;7b3e4e8e12ef:46811 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T15:25:31,131 INFO [M:0;7b3e4e8e12ef:46811 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=758.17 KB heapSize=931.81 KB 2024-11-07T15:25:31,153 DEBUG [M:0;7b3e4e8e12ef:46811 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/489a2434b38f477d89f7caa78fb05961 is 82, key is hbase:meta,,1/info:regioninfo/1730992955721/Put/seqid=0 2024-11-07T15:25:31,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742469_1645 (size=5672) 2024-11-07T15:25:31,496 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T15:25:31,497 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T15:25:31,498 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T15:25:31,556 INFO [M:0;7b3e4e8e12ef:46811 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2142 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/489a2434b38f477d89f7caa78fb05961 2024-11-07T15:25:31,584 DEBUG [M:0;7b3e4e8e12ef:46811 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c28876f09b5a4774af0f75c6da20b1db is 2275, key is \x00\x00\x00\x00\x00\x00\x00#/proc:d/1730992986977/Put/seqid=0 2024-11-07T15:25:31,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742470_1646 (size=44475) 2024-11-07T15:25:31,988 INFO [M:0;7b3e4e8e12ef:46811 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=757.62 KB at sequenceid=2142 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c28876f09b5a4774af0f75c6da20b1db 2024-11-07T15:25:31,991 INFO [M:0;7b3e4e8e12ef:46811 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c28876f09b5a4774af0f75c6da20b1db 2024-11-07T15:25:32,004 DEBUG [M:0;7b3e4e8e12ef:46811 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0fe507e6950849a588566e0583bfe5e4 is 69, key is 7b3e4e8e12ef,33849,1730992951906/rs:state/1730992954249/Put/seqid=0 2024-11-07T15:25:32,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073742471_1647 (size=5156) 2024-11-07T15:25:32,407 INFO [M:0;7b3e4e8e12ef:46811 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2142 (bloomFilter=true), to=hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0fe507e6950849a588566e0583bfe5e4 2024-11-07T15:25:32,411 DEBUG [M:0;7b3e4e8e12ef:46811 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/489a2434b38f477d89f7caa78fb05961 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/489a2434b38f477d89f7caa78fb05961 2024-11-07T15:25:32,414 INFO [M:0;7b3e4e8e12ef:46811 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/489a2434b38f477d89f7caa78fb05961, entries=8, sequenceid=2142, filesize=5.5 K 2024-11-07T15:25:32,415 DEBUG [M:0;7b3e4e8e12ef:46811 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c28876f09b5a4774af0f75c6da20b1db as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c28876f09b5a4774af0f75c6da20b1db 2024-11-07T15:25:32,418 INFO [M:0;7b3e4e8e12ef:46811 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c28876f09b5a4774af0f75c6da20b1db 2024-11-07T15:25:32,418 INFO [M:0;7b3e4e8e12ef:46811 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c28876f09b5a4774af0f75c6da20b1db, entries=172, sequenceid=2142, filesize=43.4 K 2024-11-07T15:25:32,419 DEBUG [M:0;7b3e4e8e12ef:46811 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0fe507e6950849a588566e0583bfe5e4 as hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0fe507e6950849a588566e0583bfe5e4 2024-11-07T15:25:32,422 INFO [M:0;7b3e4e8e12ef:46811 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41161/user/jenkins/test-data/e2b408fd-706e-6d45-6802-7afe47fe7938/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0fe507e6950849a588566e0583bfe5e4, entries=1, sequenceid=2142, filesize=5.0 K 2024-11-07T15:25:32,422 INFO [M:0;7b3e4e8e12ef:46811 {}] regionserver.HRegion(3140): Finished flush of dataSize ~758.17 KB/776367, heapSize ~931.52 KB/953872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1291ms, sequenceid=2142, compaction requested=false 2024-11-07T15:25:32,424 INFO [M:0;7b3e4e8e12ef:46811 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T15:25:32,424 DEBUG [M:0;7b3e4e8e12ef:46811 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730993131130Disabling compacts and flushes for region at 1730993131130Disabling writes for close at 1730993131131 (+1 ms)Obtaining lock to block concurrent updates at 1730993131131Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1730993131131Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=776367, getHeapSize=954112, getOffHeapSize=0, getCellsCount=2339 at 1730993131132 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1730993131133 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1730993131133Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1730993131153 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1730993131153Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1730993131561 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1730993131583 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1730993131583Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1730993131991 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1730993132003 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1730993132003Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@234f44b2: reopening flushed file at 1730993132411 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ea09005: reopening flushed file at 1730993132414 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60cd9d21: reopening flushed file at 1730993132418 (+4 ms)Finished flush of dataSize ~758.17 KB/776367, heapSize ~931.52 KB/953872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1291ms, sequenceid=2142, compaction requested=false at 1730993132422 (+4 ms)Writing region close event to WAL at 1730993132423 (+1 ms)Closed at 1730993132423 2024-11-07T15:25:32,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44095 is added to blk_1073741830_1006 (size=916804) 2024-11-07T15:25:32,426 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T15:25:32,426 INFO [M:0;7b3e4e8e12ef:46811 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-07T15:25:32,427 INFO [M:0;7b3e4e8e12ef:46811 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46811 2024-11-07T15:25:32,427 INFO [M:0;7b3e4e8e12ef:46811 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T15:25:32,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T15:25:32,590 INFO [M:0;7b3e4e8e12ef:46811 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T15:25:32,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46811-0x10115e447f30000, quorum=127.0.0.1:54223, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T15:25:32,634 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dfb1097{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T15:25:32,637 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T15:25:32,637 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T15:25:32,637 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T15:25:32,638 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/hadoop.log.dir/,STOPPED} 2024-11-07T15:25:32,641 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T15:25:32,641 WARN [BP-750064135-172.17.0.2-1730992947237 heartbeating to localhost/127.0.0.1:41161 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T15:25:32,641 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T15:25:32,641 WARN [BP-750064135-172.17.0.2-1730992947237 heartbeating to localhost/127.0.0.1:41161 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-750064135-172.17.0.2-1730992947237 (Datanode Uuid 2bee1f03-7b8c-47c3-83e2-8866e9b3fe97) service to localhost/127.0.0.1:41161 2024-11-07T15:25:32,644 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/cluster_df3b216e-6ff7-ba4d-e7a0-e36a33a739e6/data/data1/current/BP-750064135-172.17.0.2-1730992947237 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T15:25:32,644 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/cluster_df3b216e-6ff7-ba4d-e7a0-e36a33a739e6/data/data2/current/BP-750064135-172.17.0.2-1730992947237 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T15:25:32,645 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T15:25:32,654 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T15:25:32,655 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T15:25:32,655 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T15:25:32,655 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T15:25:32,656 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8af04474-2c7e-07f9-b631-f8fb9d434ab2/hadoop.log.dir/,STOPPED} 2024-11-07T15:25:32,672 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-07T15:25:32,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down